From 191f734cab2bde35903a54a58a94eb6889ada28c Mon Sep 17 00:00:00 2001
From: Dandandan <danielheres@gmail.com>
Date: Tue, 9 Jun 2020 18:51:06 +0200
Subject: [PATCH 1/3] Change pattern of tokenize errors to always include
 column and line number in error

---
 src/tokenizer.rs | 54 +++++++++++++++++++++++++++---------------------
 1 file changed, 30 insertions(+), 24 deletions(-)

diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 1f33dd5dc..3181c9945 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -217,7 +217,11 @@ impl fmt::Display for Whitespace {
 
 /// Tokenizer error
 #[derive(Debug, PartialEq)]
-pub struct TokenizerError(String);
+pub struct TokenizerError {
+    message: String,
+    line: u64,
+    col: u64,
+}
 
 /// SQL Tokenizer
 pub struct Tokenizer<'a> {
@@ -331,10 +335,10 @@ impl<'a> Tokenizer<'a> {
                     if chars.next() == Some(quote_end) {
                         Ok(Some(Token::make_word(&s, Some(quote_start))))
                     } else {
-                        Err(TokenizerError(format!(
-                            "Expected close delimiter '{}' before EOF.",
-                            quote_end
-                        )))
+                        self.tokenizer_error(
+                            format!("Expected close delimiter '{}' before EOF.", quote_end)
+                                .as_str(),
+                        )
                     }
                 }
                 // numbers
@@ -395,10 +399,7 @@ impl<'a> Tokenizer<'a> {
                     chars.next(); // consume
                     match chars.peek() {
                         Some('=') => self.consume_and_return(chars, Token::Neq),
-                        _ => Err(TokenizerError(format!(
-                            "Tokenizer Error at Line: {}, Col: {}",
-                            self.line, self.col
-                        ))),
+                        _ => self.tokenizer_error("Expected to see'=' after '!' character"),
                     }
                 }
                 '<' => {
@@ -437,6 +438,14 @@ impl<'a> Tokenizer<'a> {
         }
     }
 
+    fn tokenizer_error<R>(&self, message: &str) -> Result<R, TokenizerError> {
+        Err(TokenizerError {
+            message: message.to_string(),
+            col: self.col,
+            line: self.line,
+        })
+    }
+
     /// Tokenize an identifier or keyword, after the first char is already consumed.
     fn tokenize_word(&self, first_char: char, chars: &mut Peekable<Chars<'_>>) -> String {
         let mut s = first_char.to_string();
@@ -471,10 +480,7 @@ impl<'a> Tokenizer<'a> {
                 }
             }
         }
-        Err(TokenizerError(format!(
-            "Unterminated string literal at Line: {}, Col: {}",
-            self.line, self.col
-        )))
+        self.tokenizer_error("Unterminated string literal")
     }
 
     fn tokenize_multiline_comment(
@@ -499,11 +505,7 @@ impl<'a> Tokenizer<'a> {
                         s.push(ch);
                     }
                 }
-                None => {
-                    break Err(TokenizerError(
-                        "Unexpected EOF while in a multi-line comment".to_string(),
-                    ));
-                }
+                None => break self.tokenizer_error("Unexpected EOF while in a multi-line comment"),
             }
         }
     }
@@ -720,9 +722,11 @@ mod tests {
         let mut tokenizer = Tokenizer::new(&dialect, &sql);
         assert_eq!(
             tokenizer.tokenize(),
-            Err(TokenizerError(
-                "Unterminated string literal at Line: 1, Col: 8".to_string()
-            ))
+            Err(TokenizerError {
+                message: "Unterminated string literal".to_string(),
+                line: 1,
+                col: 8
+            })
         );
     }
 
@@ -843,9 +847,11 @@ mod tests {
         let mut tokenizer = Tokenizer::new(&dialect, &sql);
         assert_eq!(
             tokenizer.tokenize(),
-            Err(TokenizerError(
-                "Expected close delimiter '\"' before EOF.".to_string(),
-            ))
+            Err(TokenizerError {
+                message: "Expected close delimiter '\"' before EOF.".to_string(),
+                line: 1,
+                col: 1
+            })
         );
     }
 

From be1279be9afa3ae42c5f9fab0926eb29e2a23b32 Mon Sep 17 00:00:00 2001
From: Dandandan <danielheres@gmail.com>
Date: Tue, 9 Jun 2020 18:53:27 +0200
Subject: [PATCH 2/3] Add a missing space in description

---
 src/tokenizer.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 3181c9945..90315bf36 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -399,7 +399,7 @@ impl<'a> Tokenizer<'a> {
                     chars.next(); // consume
                     match chars.peek() {
                         Some('=') => self.consume_and_return(chars, Token::Neq),
-                        _ => self.tokenizer_error("Expected to see'=' after '!' character"),
+                        _ => self.tokenizer_error("Expected to see '=' after '!' character"),
                     }
                 }
                 '<' => {

From ec7a0948a6c68e9e4b18f483d5aadb28f9e6fe2e Mon Sep 17 00:00:00 2001
From: Dandandan <danielheres@gmail.com>
Date: Tue, 9 Jun 2020 19:28:26 +0200
Subject: [PATCH 3/3] Add location info to ParserError::TokenizerError

---
 src/parser.rs    | 5 ++++-
 src/tokenizer.rs | 6 +++---
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/parser.rs b/src/parser.rs
index 9a22f4d19..780334a9b 100644
--- a/src/parser.rs
+++ b/src/parser.rs
@@ -49,7 +49,10 @@ use IsLateral::*;
 
 impl From<TokenizerError> for ParserError {
     fn from(e: TokenizerError) -> Self {
-        ParserError::TokenizerError(format!("{:?}", e))
+        ParserError::TokenizerError(format!(
+            "{} at Line: {}, Column {}",
+            e.message, e.line, e.col
+        ))
     }
 }
 
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 90315bf36..68ebfaf62 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -218,9 +218,9 @@ impl fmt::Display for Whitespace {
 /// Tokenizer error
 #[derive(Debug, PartialEq)]
 pub struct TokenizerError {
-    message: String,
-    line: u64,
-    col: u64,
+    pub message: String,
+    pub line: u64,
+    pub col: u64,
 }
 
 /// SQL Tokenizer