feat(lexer): Add diagnostic and impl error recovering

src/frontend/lexer.rs

@@ -1,21 +1,32 @@
-use std::{io::BufRead, iter::Peekable, str::FromStr};
+use std::{io::BufRead, str::FromStr};
 
+use codespan_reporting::diagnostic;
 use thiserror::Error;
 
-use crate::frontend::types::{Span, TokenValue, TypeIdent};
+use crate::{diagnostic::{Diagnositics, span::{self, Span}}, frontend::{err::LexingError, types::{TokenValue, TypeIdent}}};
 
 use super::types::Token;
 
 pub struct Lexer {
-    tokens: Vec<Token>,
-    errors: Vec<usize>, // every entry points to the index of unrecognized tokens
+    pub tokens: Vec<Token>,
+    pub diagnostics: Diagnositics,
+    old_char_count: usize,
+    block_comment_span: Option<Span>,
+    in_skip_line: bool
 }
 
 const WHITESPACE_CHARS: &[char] = &[' ', '\t', '\n', '\r'];
+const DELIMITER_CHARS: &[char] = &[
+    '+', '-', '*', '/', '%', '=', '!', '<', '>', '(', ')', ',', ';'
+];
 struct Cursor {
     chars: Vec<char>,
     pos: usize,
 }
+enum LexParseError {
+    NotMatched,
+    InvalidInMatch(LexingError)
+}
 impl Cursor {
     pub fn new(s: &str) -> Self {
         Self { chars: s.chars().collect(), pos: 0 }
@@ -47,20 +58,35 @@ impl Cursor {
         self.pos
     }
 }
+/// try parse using the giving function, return whether should continue
 fn try_parse_as(
-    f: fn(&mut Cursor) -> Option<TokenValue>,
+    f: fn(&mut Cursor) -> Result<TokenValue, LexParseError>,
     tokens: &mut Vec<Token>,
     str_iter: &mut Cursor,
-    line: &mut usize,
-    column: &mut usize,
+    diagnostics: &mut Diagnositics,
+    last_char_count: usize,
 ) -> bool {
-    let last_pos = str_iter.pos();
-    if let Some(token) = f(str_iter) {
-        let span = Span { line: *line, column: *column, length: str_iter.pos() - last_pos };
-        tokens.push(Token { value: token, span });
-        return true;
+    let last_pos = str_iter.pos() + last_char_count;
+    match f(str_iter) {
+        Ok(token_value) => {
+            let span = Span { start: last_pos, end: str_iter.pos() + last_char_count };
+            tokens.push(Token { value: token_value, span });
+            return true;
+        }
+        Err(LexParseError::NotMatched) => false,
+        Err(LexParseError::InvalidInMatch(err)) => {
+            // try recover from delimiter char or whitespace char
+            while let Some(c) = str_iter.peek() {
+                if DELIMITER_CHARS.contains(&c) || WHITESPACE_CHARS.contains(&c) {
+                    break;
+                }
+                str_iter.advance(1);
+            }
+            let span = Span { start: last_pos, end: str_iter.pos() + last_char_count };
+            diagnostics.add_from_frontend_error(err, span);
+            return true;
+        }
     }
-    false
 }
 macro_rules! if_true_then_continue {
     ($e: expr) => {
@@ -77,99 +103,106 @@ pub enum LexerError {
     TooManyErrors,
 }
 impl Lexer {
-    pub fn has_errors(&self) -> bool {
-        !self.errors.is_empty()
+    pub fn new() -> Self {
+        Self { tokens: vec![], diagnostics: Diagnositics::new(), old_char_count: 0, block_comment_span: None, in_skip_line: false }
     }
-    pub fn parse(reader: &mut impl BufRead) -> Result<Self, LexerError> {
-        let mut tokens = Vec::new();
-        let mut errors = Vec::new();
-        let mut line = 1;
-        let mut column = 1;
-        let mut in_block_comment = false;
-        for line_str in reader.lines() {
-            let line_str = line_str?;
-            let mut cursor = Cursor::new(&line_str);
-                loop {
-                    if let Some(c) = cursor.peek() {
-                        // check white space first, if it's white space, skip it and continue to the next character
-                        if WHITESPACE_CHARS.contains(&c) {
-                            column += 1;
-                            cursor.advance(1);
-                            continue;
-                        }
-                        // check comment
-                        match cursor.peek_multiple(2) {
-                            Some(['/', '/']) => {
-                                // skip the rest of the line
-                                line += 1;
-                                column = 1;
-                                break;
-                            }
-                            Some(['/', '*']) => {
-                                in_block_comment = true;
-                                cursor.advance(2);
-                                column += 2;
-                                continue;
-                            }
-                            Some(['*', '/']) => {
-                                in_block_comment = false;
-                                cursor.advance(2);
-                                column += 2;
-                                continue;
-                            }
-                            _ => {}
-                        }
-                    } else {
-                        break;
-                    }
-                    if in_block_comment {
-                        cursor.advance(1);
-                        column += 1;
-                    }
-                    if_true_then_continue!(try_parse_as(parse_litint, &mut tokens, &mut cursor, &mut line, &mut column));
-                    if_true_then_continue!(try_parse_as(parse_delimiter, &mut tokens, &mut cursor, &mut line, &mut column));
-                    if_true_then_continue!(try_parse_as(parse_puncuation, &mut tokens, &mut cursor, &mut line, &mut column));
-                    if_true_then_continue!(try_parse_as(parse_ident, &mut tokens, &mut cursor, &mut line, &mut column));
-                    // unrecognized token
-                    errors.push(tokens.len());
-                    let c = cursor.next().unwrap();
-                    tokens.push(Token {
-                        value: TokenValue::Unrecognized(c),
-                        span: Span { line, column, length: 1 },
-                    });
-                    if errors.len() > 20 {
-                        return Err(LexerError::TooManyErrors);
-                    }
-                    column += 1;
-                }
-            line += 1;
-            column = 1;
+    pub fn finish(mut self) -> (Vec<Token>, Diagnositics) {
+        if let Some(span) = self.block_comment_span.take() {
+            self.diagnostics.add_from_frontend_error(LexingError::UnterminatedComment, span);
         }
-        Ok(Self { tokens, errors })
+        (self.tokens, self.diagnostics)
+    }
+    /// call `parse_str` will continue to parse the input from current state
+    /// please also pass the whitespace to ensure the correct char position in diagnostics
+    pub fn parse_next_str(&mut self, s: &str) {
+        let mut cursor = Cursor::new(s);
+        loop {
+            if let Some(c) = cursor.peek() {
+                if self.in_skip_line && c != '\n' {
+                    cursor.advance(1);
+                    continue;
+                }
+                // check white space first, if it's white space, skip it and continue to the next character
+                if WHITESPACE_CHARS.contains(&c) {
+                    if c == '\n' {
+                        self.in_skip_line = false;
+                    }
+                    cursor.advance(1);
+                    continue;
+                }
+                // check comment
+                match cursor.peek_multiple(2) {
+                    Some(['/', '/']) => {
+                        // skip the rest of the line
+                        self.in_skip_line = true;
+                        cursor.advance(2);
+                        continue;
+                    }
+                    Some(['/', '*']) => {
+                        let start = cursor.pos() + self.old_char_count;
+                        self.block_comment_span = Some(Span { start, end: start + 2 });
+                        cursor.advance(2);
+                        continue;
+                    }
+                    Some(['*', '/']) => {
+                        self.block_comment_span = None;
+                        cursor.advance(2);
+                        continue;
+                    }
+                    _ => {}
+                }
+                if self.block_comment_span.is_some() {
+                    cursor.advance(1);
+                    continue;
+                }
+            } else {
+                break;
+            }
+            if_true_then_continue!(try_parse_as(parse_litint, &mut self.tokens, &mut cursor, &mut self.diagnostics, self.old_char_count));
+            if_true_then_continue!(try_parse_as(parse_delimiter, &mut self.tokens, &mut cursor, &mut self.diagnostics, self.old_char_count));
+            if_true_then_continue!(try_parse_as(parse_puncuation, &mut self.tokens, &mut cursor, &mut self.diagnostics, self.old_char_count));
+            if_true_then_continue!(try_parse_as(parse_ident, &mut self.tokens, &mut cursor, &mut self.diagnostics, self.old_char_count));
+            // unrecognized token
+            let last_pos = cursor.pos() + self.old_char_count;
+            let mut unrecognized = Vec::new();
+            while let Some(c) = cursor.peek() {
+                if DELIMITER_CHARS.contains(&c) || WHITESPACE_CHARS.contains(&c) {
+                    break;
+                }
+                unrecognized.push(c);
+                cursor.advance(1);
+            }
+            let span = Span { start: last_pos, end: cursor.pos() + self.old_char_count };
+            let unrecognized = unrecognized.into_iter().collect::<String>();
+            self.diagnostics.add_from_frontend_error(LexingError::UnrecognizedToken(unrecognized), span);
+            self.tokens.push(Token { value: TokenValue::Unrecognized, span });
+        }
+        self.old_char_count += s.len();
     }
 }
 fn parse_litint(
     str_iter: &mut Cursor,
-) -> Option<TokenValue> {
-    let mut c1 = str_iter.peek()?;
+) -> Result<TokenValue, LexParseError> {
+    let mut c1 = str_iter.peek().ok_or(LexParseError::NotMatched)?;
     // c1 is the peek value from here
     let mut sign_base: i64 = 1;
     let mut base: i64 = 10;
     if !(c1.is_ascii_digit() || c1 == '-') {
-        return None;
+        return Err(LexParseError::NotMatched);
     }
     if c1 == '-' {
         sign_base = -1;
         str_iter.advance(1);
-        c1 = str_iter.peek()?;
+        c1 = str_iter.peek().ok_or(LexParseError::NotMatched)?;
         if !c1.is_ascii_digit() {
             // only a minus sign, not a number
             // back one so cursor still points to the minus sign
             str_iter.back(1);
-            return None;
+            return Err(LexParseError::NotMatched);
         }
     }
     let mut number = 0i64;
+    let mut has_digits = false;
     if c1 == '0' {
         str_iter.advance(1);
         match str_iter.peek() {
@@ -181,12 +214,13 @@ fn parse_litint(
                 base = 8;
             }
             _ => {
+                has_digits = true;
                 // only zero
             }
         }
     }
     // from here, the cursor points to:
-    // 0x1234 -> cursor at 'x'
+    // 0x1234 -> cursor at '1'
     // 0123 -> cursor at '1'
     // 0 -> cursor at end
     // 1234 -> cursor at '1'
@@ -199,32 +233,42 @@ fn parse_litint(
             '0'..='9' if (c as u8 - b'0') < base as u8 => c as i64 - '0' as i64 ,
             'a'..='f' if base == 16 => c as i64 - 'a' as i64 + 10 ,
             'A'..='F' if base == 16 => c as i64 - 'A' as i64 + 10,
-            _ => break,
+            c => if WHITESPACE_CHARS.contains(&c) || DELIMITER_CHARS.contains(&c) {
+                break;
+            } else {
+                // unrecognized character in number literal
+                return Err(LexParseError::InvalidInMatch(LexingError::InvalidIntLiteral));
+            }
         };
+        has_digits = true;
         number = number * base + digit;
         str_iter.advance(1);
     }
+    if !has_digits {
+        // No valid digits found, add a diagnostic
+        return Err(LexParseError::InvalidInMatch(LexingError::InvalidIntLiteral));
+    }
     number *= sign_base;
-    Some(TokenValue::IntLit(number))
+    Ok(TokenValue::IntLit(number))
 }
 
 fn parse_delimiter(
     str_iter: &mut Cursor,
-) -> Option<TokenValue> {
-    let c = str_iter.peek()?;
+) -> Result<TokenValue, LexParseError> {
+    let c = str_iter.peek().ok_or(LexParseError::NotMatched)?;
     let token_value = match c {
         '(' => TokenValue::LParen,
         ')' => TokenValue::RParen,
         '{' => TokenValue::LBrace,
         '}' => TokenValue::RBrace,
-        _ => return None,
+        _ => return Err(LexParseError::NotMatched),
     };
     str_iter.advance(1);
-    Some(token_value)
+    Ok(token_value)
 }
 fn parse_puncuation(
     str_iter: &mut Cursor,
-) -> Option<TokenValue> {
+) -> Result<TokenValue, LexParseError> {
     let get_value_by_next_char = 
         |str_iter: &mut Cursor, not_equal_value: TokenValue, equal_value: TokenValue| {
             str_iter.advance(1);
@@ -235,7 +279,7 @@ fn parse_puncuation(
                 not_equal_value
             }
     };
-    let c = str_iter.peek()?;
+    let c = str_iter.peek().ok_or(LexParseError::NotMatched)?;
     let token_value = match c {
         '+' => TokenValue::Plus,
         '-' => TokenValue::Minus,
@@ -249,9 +293,7 @@ fn parse_puncuation(
             if let Some('=') = str_iter.peek() {
                 TokenValue::NotEqual
             } else {
-                // only '!' is not a valid token, back one so cursor still points to '!'
-                str_iter.back(1);
-                return None;
+                TokenValue::Not
             }
         },
         '<' => get_value_by_next_char(str_iter, TokenValue::Less, TokenValue::LessEqual),
@@ -260,33 +302,35 @@ fn parse_puncuation(
         ',' => TokenValue::Comma,
         ';' => TokenValue::Semicolon,
 
-        _ => return None,
+        _ => return Err(LexParseError::NotMatched),
     };
     str_iter.advance(1);
-    Some(token_value)
+    Ok(token_value)
 }
 
 fn parse_ident(
     str_iter: &mut Cursor,
-) -> Option<TokenValue> {
-    let c = str_iter.peek()?;
+) -> Result<TokenValue, LexParseError> {
+    let c = str_iter.peek().ok_or(LexParseError::NotMatched)?;
     if !c.is_ascii_alphabetic() && c != '_' {
-        return None;
+        return Err(LexParseError::NotMatched);
     }
     let mut name = Vec::new();
     while let Some(c) = str_iter.peek() {
         if c.is_ascii_alphanumeric() || c == '_' {
             name.push(c);
             str_iter.advance(1);
-        } else {
+        } else if DELIMITER_CHARS.contains(&c) || WHITESPACE_CHARS.contains(&c) {
             break;
+        } else {
+            return Err(LexParseError::InvalidInMatch(LexingError::InvalidIdent));
         }
     }
     let name = name.into_iter().collect::<String>();
     if let Some(type_ident) = TypeIdent::from_str(&name).ok() {
-        return Some(TokenValue::TypeIdent(type_ident));
+        return Ok(TokenValue::TypeIdent(type_ident));
     }
-    Some(TokenValue::Ident(name))
+    Ok(TokenValue::Ident(name))
 }
 #[cfg(test)]
 mod tests {
@@ -303,11 +347,22 @@ mod tests {
         for case_no in case_sequence {
             let case_path = case_list.get_case_path(case_no).unwrap();
             println!("{}", case_path.display());
-            let file = File::open(case_path).unwrap();
+            let file = File::open(&case_path).unwrap();
             let mut buf_reader = std::io::BufReader::new(file);
-            let lexer = Lexer::parse(&mut buf_reader).unwrap();
-            if lexer.has_errors() {
-                eprintln!("Case {} has error", case_list.get_case_name(case_no).unwrap());
+            let mut lexer = Lexer::new();
+            let mut full_text = String::new();
+            loop {
+                let mut line = String::new();
+                let bytes_read = buf_reader.read_line(&mut line).unwrap();
+                if bytes_read == 0 {
+                    break;
+                }
+                full_text.push_str(&line);
+                lexer.parse_next_str(&line);
+            }
+            let (_tokens, diagnostics) = lexer.finish();
+            if !diagnostics.is_empty() {
+                diagnostics.print(&format!("{}", case_path.display()), &full_text);
                 error_case_cnt += 1;
             }
         }