From 5099bbaab793e9a382e3dd84f556f5332c524625 Mon Sep 17 00:00:00 2001 From: Hydrostic Date: Fri, 12 Jun 2026 18:32:13 +0800 Subject: [PATCH] refactor(lexer, ast): Rewrite decl with c standard grammar of decl --- src/ast/parser/decl.rs | 614 ++++++++++++++++++++++++++--------------- src/ast/parser/stmt.rs | 2 +- src/lexer/lexer.rs | 16 +- src/lexer/types.rs | 5 +- 4 files changed, 407 insertions(+), 230 deletions(-) diff --git a/src/ast/parser/decl.rs b/src/ast/parser/decl.rs index 19edba5..9ce2eab 100644 --- a/src/ast/parser/decl.rs +++ b/src/ast/parser/decl.rs @@ -1,7 +1,8 @@ use crate::{ ast::err::ParseError, ast::types::{ - ArrayDimension, CompileUnit, FuncDeclStmt, GlobalDeclStmt, Param, VarDeclStmt, VarDeclStmtValue, + ArrayDimension, CompileUnit, Expr, FuncDeclStmt, GlobalDeclStmt, Param, VarDeclStmt, + VarDeclStmtValue, }, diagnostic::span::Span, lexer::types::{TokenValue, TypeIdent}, @@ -9,11 +10,41 @@ use crate::{ use super::{ParseProcessError, ParseType, Parser}; +#[derive(Clone, Copy)] +enum StorageClass { + Static, +} + +struct DeclSpecifiers { + _storage_class: Option, + type_specifier: TypeIdent, + type_span: Span, +} + +struct Declarator { + name: String, + name_span: Span, + suffixes: Vec, +} + +enum DeclaratorSuffix { + Array(ArrayDimension), + Function { + params: Vec, + span: Span, + }, +} + +struct InitDeclarator { + declarator: Declarator, + initializer: Option, +} + impl Parser { pub(super) fn parse_compile_unit(&mut self) -> CompileUnit { let mut global_decls = vec![]; while !self.is_eof() { - if let Some(decl) = self.parse_global_decl_stmt() { + if let Some(decl) = self.parse_external_decl_stmt() { global_decls.push(decl); } else { self.until_next_token(&[TokenValue::Semicolon, TokenValue::RBrace]); @@ -22,260 +53,391 @@ impl Parser { CompileUnit { global_decls } } - fn parse_global_decl_stmt(&mut self) -> Option { - match self.parse_func_decl_stmt() { - Ok(func_decl) => return Some(GlobalDeclStmt::FuncDecl(func_decl)), - Err(ParseProcessError::ErrorInMatch) => { - return None - }, - _ => {} + fn parse_external_decl_stmt(&mut self) -> Option { + let specifiers = match self.parse_decl_specifiers(ParseType::MustParse) { + Ok(specifiers) => specifiers, + Err(ParseProcessError::ErrorInMatch) => return None, + Err(ParseProcessError::TryNext) => unreachable!(), }; - match self.parse_var_decl_stmt(ParseType::MustParse) { - Ok(var_decl) => return Some(GlobalDeclStmt::VarDecl(var_decl)), - Err(ParseProcessError::ErrorInMatch) => { - return None - }, - _ => {} - } - let token = self.next().clone(); - self.diagnostics.add_from_frontend_error( - ParseError::ExpectedBefore(token.value, "ident"), - token.span, - ); - None - } - // fn until_next_stmt(&mut self) { - // // skip tokens until we find a semicolon or a right brace, which may indicate the end of the declaration - // while let Some(t) = self.peek() { - // if matches!(t.value, TokenValue::Semicolon | TokenValue::RBrace) { - // self.advance(1); - // break; - // } - // self.advance(1); - // } - // } - fn parse_type_and_name(&mut self, parse_type: ParseType) -> Result<(TypeIdent, String, Span, Span), ParseProcessError> { - let type_token = self.peek().clone(); - let type_ident = match self.peek().value.as_type_ident() { - Some(ti) => ti, - None => { - if matches!(parse_type, ParseType::MustParse) { - let token = self.next().clone(); - self.diagnostics.add_from_frontend_error( - ParseError::ExpectedBefore(token.value, "type ident"), - token.span, - ); - return Err(ParseProcessError::ErrorInMatch); - } - return Err(ParseProcessError::TryNext); - }, + let declarator = match self.parse_declarator() { + Ok(declarator) => declarator, + Err(ParseProcessError::ErrorInMatch) => return None, + Err(ParseProcessError::TryNext) => unreachable!(), }; - let type_span = type_token.span; - self.advance(1); - let name = match self.peek().value.as_ident() { - None => { - let next_span = self.peek().span; - self.diagnostics.add_from_frontend_error( - ParseError::ExpectedIdentAfter(type_token.value), - next_span - ); - return Err(ParseProcessError::ErrorInMatch); - } - Some(ident) => ident, - }; - let name_span = self.peek().span; - self.advance(1); - Ok((type_ident, name, type_span, name_span)) - } - fn parse_func_decl_stmt(&mut self) -> Result { - let (return_type, name, ret_type_span, name_span) = self.parse_type_and_name(ParseType::MustParse)?; - if matches!(self.peek().value, TokenValue::LParen) { - } else { - self.back(2); - return Err(ParseProcessError::TryNext); - } - // from here we can be sure it's a function declaration, so we can report error if the syntax is wrong - let params = self.parse_param_list()?; - let body = self.parse_block_stmt(ParseType::MustParse)?; - Ok(FuncDeclStmt { - return_type: return_type.into(), - name, - params, - body, - ret_type_span, - name_span, - }) - } - fn parse_param_list(&mut self) -> Result, ParseProcessError> { - if self.peek().value != TokenValue::LParen { - let token = self.next().clone(); - self.diagnostics.add_from_frontend_error( - ParseError::ExpectedBefore(token.value, "`(`"), - token.span, - ); - return Err(ParseProcessError::ErrorInMatch); - } - self.advance(1); - let mut params = vec![]; - let mut last_is_var = false; - while !self.is_eof() { - if self.peek().value == TokenValue::RParen { - self.advance(1); - break; - } - if last_is_var { - self.must_match_token(&TokenValue::Comma, "`,` or `)`") - .inspect_err(|_| self.until_next_token(&[TokenValue::RBrace]))?; - } - match self.parse_param() { - Ok(param) => { - params.push(param); - last_is_var = true; - } - Err(_e) => { - self.until_next_token(&[TokenValue::RParen]); - if self.last().value == TokenValue::RParen { - break; - } - } + + if self.peek().value == TokenValue::LBrace { + match self.finish_func_def_stmt(specifiers, declarator) { + Ok(func_decl) => return Some(GlobalDeclStmt::FuncDecl(func_decl)), + Err(ParseProcessError::ErrorInMatch) => return None, + Err(ParseProcessError::TryNext) => unreachable!(), } } - Ok(params) - } - fn parse_param(&mut self) -> Result { - let (param_type, name, type_span, name_span) = self.parse_type_and_name(ParseType::MustParse)?; - Ok(Param { - param_type: param_type.into(), - name, - dimensions: self.parse_array_dimensions(true)?, - name_span, - type_span, - }) - } - // fn must_match_semicolon(&mut self) -> Option<()> { - // if self - // .peek() - // .is_some_and(|t| matches!(t.value, TokenValue::Semicolon)) - // { - // self.advance(1); - // Some(()) - // } else { - // let token = self.next().clone(); - // self.diagnostics - // .add_from_frontend_error(ParseError::ExpectedBefore(token.value, "`;`"), token.span); - // while let Some(t) = self.peek() { - // if matches!(t.value, TokenValue::Semicolon) { - // self.advance(1); - // break; - // } - // if matches!(t.value, TokenValue::RBrace) { - // break; - // } - // self.advance(1); - // } - // None - // } - // } - pub(super) fn parse_var_decl_stmt(&mut self, parse_type: ParseType) -> Result { - self.parse_var_decl_stmt_inner(parse_type, true) + + match self.finish_var_decl_stmt(specifiers, declarator, true, true) { + Ok(var_decl) => Some(GlobalDeclStmt::VarDecl(var_decl)), + Err(ParseProcessError::ErrorInMatch) => None, + Err(ParseProcessError::TryNext) => unreachable!(), + } } - pub(super) fn parse_var_decl_stmt_inner(&mut self, parse_type: ParseType, consume_semicolon: bool) -> Result { - let mut values = vec![]; - let (var_type, name, type_span, name_span) = match self.parse_type_and_name(parse_type) { - Ok(res) => res, - Err(_e) => { - if matches!(parse_type, ParseType::TryParse) { - return Err(ParseProcessError::TryNext); - } else { - return Err(ParseProcessError::ErrorInMatch); - } - } - }; - let dimensions = self.parse_array_dimensions(false)?; - let value = if self.peek().value == TokenValue::Equal { + fn parse_decl_specifiers( + &mut self, + parse_type: ParseType, + ) -> Result { + let storage_class = if self.peek().value == TokenValue::Static { self.advance(1); - Some(self.parse_expr()?) + Some(StorageClass::Static) } else { None }; - values.push(VarDeclStmtValue { name, name_span, dimensions, value }); - let mut last_name = true; // indicate whether the last parsed token is a variable name - while !self.is_eof() { - if matches!(self.peek().value, TokenValue::Semicolon) { // statement end - break; + + let type_token = self.peek().clone(); + let type_specifier = match type_token.value.as_type_ident() { + Some(type_specifier) => { + self.advance(1); + type_specifier } - if last_name { // expect a comma after a variable name - self.must_match_token(&TokenValue::Comma, "`,` or `;`")?; - last_name = false; + None if parse_type == ParseType::TryParse && storage_class.is_none() => { + return Err(ParseProcessError::TryNext); } - if let Some(ident) = self.peek().value.as_ident() { - let span = self.next().span; - let dimensions = self.parse_array_dimensions(false)?; - let value = if self.peek().value == TokenValue::Equal { - self.advance(1); - Some(self.parse_expr()?) - } else { - None - }; - values.push(VarDeclStmtValue { name: ident, name_span: span, dimensions, value }); - last_name = true; - } else { - let token = self.next().clone(); + None => { + let token = self.next(); self.diagnostics.add_from_frontend_error( - ParseError::ExpectedIdentAfter(TokenValue::TypeIdent(var_type)), + ParseError::ExpectedBefore(token.value, "type specifier"), token.span, ); return Err(ParseProcessError::ErrorInMatch); } + }; + + Ok(DeclSpecifiers { + _storage_class: storage_class, + type_specifier, + type_span: type_token.span, + }) + } + + fn parse_declarator(&mut self) -> Result { + self.parse_direct_declarator() + } + + fn parse_direct_declarator(&mut self) -> Result { + let mut declarator = match self.peek().value.clone() { + TokenValue::Ident(name) => { + let name_span = self.peek().span; + self.advance(1); + Declarator { + name, + name_span, + suffixes: vec![], + } + } + TokenValue::LParen => { + self.advance(1); + let declarator = self.parse_declarator()?; + self.must_match_token(&TokenValue::RParen, "`)`")?; + declarator + } + _ => { + let token = self.next(); + self.diagnostics.add_from_frontend_error( + ParseError::ExpectedBefore(token.value, "declarator"), + token.span, + ); + return Err(ParseProcessError::ErrorInMatch); + } + }; + + loop { + match self.peek().value { + TokenValue::LBracket => { + let dimension = self.parse_array_declarator_suffix()?; + declarator.suffixes.push(DeclaratorSuffix::Array(dimension)); + } + TokenValue::LParen => { + let start_span = self.next().span; + let params = self.parse_parameter_list_after_lparen()?; + let end_span = self.last().span; + declarator.suffixes.push(DeclaratorSuffix::Function { + params, + span: Span::from_two(start_span, end_span), + }); + } + _ => break, + } } + + Ok(declarator) + } + + fn parse_array_declarator_suffix(&mut self) -> Result { + let start_span = self.next().span; + let value = if self.peek().value == TokenValue::RBracket { + None + } else { + Some(self.parse_expr()?) + }; + let end_span = match &self.peek().value { + TokenValue::RBracket => { + let span = self.peek().span; + self.advance(1); + span + } + TokenValue::Eof => { + self.diagnostics.add_from_frontend_error( + ParseError::ExpectedBefore(TokenValue::Eof, "`]`"), + start_span, + ); + return Err(ParseProcessError::ErrorInMatch); + } + _ => { + let token = self.next(); + self.diagnostics.add_from_frontend_error( + ParseError::ExpectedBefore(token.value, "`]`"), + token.span, + ); + return Err(ParseProcessError::ErrorInMatch); + } + }; + Ok(ArrayDimension { + value, + span: Span::from_two(start_span, end_span), + }) + } + + fn parse_parameter_list_after_lparen(&mut self) -> Result, ParseProcessError> { + if self.peek().value == TokenValue::RParen { + self.advance(1); + return Ok(vec![]); + } + + let mut params = vec![]; + loop { + match self.parse_param() { + Ok(param) => params.push(param), + Err(error) => { + self.until_next_token(&[TokenValue::Comma, TokenValue::RParen]); + if self.last().value == TokenValue::RParen { + return Err(error); + } + continue; + } + } + + match self.peek().value { + TokenValue::Comma => { + self.advance(1); + } + TokenValue::RParen => { + self.advance(1); + break; + } + _ => { + self.must_match_token(&TokenValue::RParen, "`,` or `)`")?; + break; + } + } + } + + Ok(params) + } + + fn parse_param(&mut self) -> Result { + let specifiers = self.parse_decl_specifiers(ParseType::MustParse)?; + let declarator = self.parse_declarator()?; + self.declarator_to_param(specifiers, declarator) + } + + pub(super) fn parse_var_decl_stmt( + &mut self, + parse_type: ParseType, + ) -> Result { + self.parse_var_decl_stmt_inner(parse_type, true) + } + + pub(super) fn parse_var_decl_stmt_inner( + &mut self, + parse_type: ParseType, + consume_semicolon: bool, + ) -> Result { + let specifiers = self.parse_decl_specifiers(parse_type)?; + let declarator = self.parse_declarator()?; + self.finish_var_decl_stmt(specifiers, declarator, consume_semicolon, false) + } + + fn finish_func_def_stmt( + &mut self, + specifiers: DeclSpecifiers, + declarator: Declarator, + ) -> Result { + let Declarator { + name, + name_span, + suffixes, + } = declarator; + let params = self.take_function_params(name_span, suffixes)?; + let body = self.parse_block_stmt(ParseType::MustParse)?; + Ok(FuncDeclStmt { + return_type: specifiers.type_specifier.into(), + name, + params, + body, + ret_type_span: specifiers.type_span, + name_span, + }) + } + + fn finish_var_decl_stmt( + &mut self, + specifiers: DeclSpecifiers, + first_declarator: Declarator, + consume_semicolon: bool, + allow_empty_declarator_list: bool, + ) -> Result { + let mut values = vec![]; + + if self.peek().value == TokenValue::Semicolon && allow_empty_declarator_list { + if consume_semicolon { + self.advance(1); + } + return Ok(VarDeclStmt { + values, + type_span: specifiers.type_span, + data_type: specifiers.type_specifier.into(), + }); + } + + let initializer = self.parse_initializer()?; + values.push(self.init_declarator_to_var_value(InitDeclarator { + declarator: first_declarator, + initializer, + })?); + + while self.peek().value == TokenValue::Comma { + self.advance(1); + let declarator = self.parse_declarator()?; + let initializer = self.parse_initializer()?; + values.push(self.init_declarator_to_var_value(InitDeclarator { + declarator, + initializer, + })?); + } + if consume_semicolon { self.must_match_token(&TokenValue::Semicolon, "`;`")?; } - Ok(VarDeclStmt { values, type_span, data_type: var_type.into() }) + + Ok(VarDeclStmt { + values, + type_span: specifiers.type_span, + data_type: specifiers.type_specifier.into(), + }) } - fn parse_array_dimensions(&mut self, allow_empty_first: bool) -> Result, ParseProcessError> { + fn parse_initializer(&mut self) -> Result, ParseProcessError> { + if self.peek().value != TokenValue::Equal { + return Ok(None); + } + self.advance(1); + Ok(Some(self.parse_expr()?)) + } + + fn init_declarator_to_var_value( + &mut self, + init_declarator: InitDeclarator, + ) -> Result { + let Declarator { + name, + name_span, + suffixes, + } = init_declarator.declarator; + let dimensions = self.take_array_dimensions(suffixes)?; + Ok(VarDeclStmtValue { + name, + name_span, + dimensions, + value: init_declarator.initializer, + }) + } + + fn declarator_to_param( + &mut self, + specifiers: DeclSpecifiers, + declarator: Declarator, + ) -> Result { + let Declarator { + name, + name_span, + suffixes, + } = declarator; + let dimensions = self.take_array_dimensions(suffixes)?; + Ok(Param { + param_type: specifiers.type_specifier.into(), + name, + dimensions, + name_span, + type_span: specifiers.type_span, + }) + } + + fn take_array_dimensions( + &mut self, + suffixes: Vec, + ) -> Result, ParseProcessError> { let mut dimensions = vec![]; - while self.peek().value == TokenValue::LBracket { - let start_span = self.next().span; - let value = if self.peek().value == TokenValue::RBracket { - if !allow_empty_first || !dimensions.is_empty() { - let span = self.peek().span; + for suffix in suffixes { + match suffix { + DeclaratorSuffix::Array(dimension) => dimensions.push(dimension), + DeclaratorSuffix::Function { span, .. } => { self.diagnostics.add_from_frontend_error( - ParseError::ExpectedBefore(TokenValue::RBracket, "array dimension expression"), + ParseError::ExpectedBefore(TokenValue::LParen, "variable declarator"), span, ); return Err(ParseProcessError::ErrorInMatch); } - None - } else { - Some(self.parse_expr()?) - }; - let end_span = match &self.peek().value { - TokenValue::RBracket => { - let span = self.peek().span; - self.advance(1); - span - } - TokenValue::Eof => { - self.diagnostics.add_from_frontend_error(ParseError::ExpectedBefore(TokenValue::Eof, "`]`"), start_span); - return Err(ParseProcessError::ErrorInMatch); - } - _ => { - let token = self.next(); - self.diagnostics.add_from_frontend_error( - ParseError::ExpectedBefore(token.value, "`]`"), - token.span, - ); - return Err(ParseProcessError::ErrorInMatch); - } - }; - dimensions.push(ArrayDimension { - value, - span: Span::from_two(start_span, end_span), - }); + } } Ok(dimensions) } + + fn take_function_params( + &mut self, + name_span: Span, + suffixes: Vec, + ) -> Result, ParseProcessError> { + let mut params = None; + for suffix in suffixes { + match suffix { + DeclaratorSuffix::Function { + params: suffix_params, + .. + } if params.is_none() => params = Some(suffix_params), + DeclaratorSuffix::Function { span, .. } => { + self.diagnostics.add_from_frontend_error( + ParseError::ExpectedBefore(TokenValue::LParen, "function body"), + span, + ); + return Err(ParseProcessError::ErrorInMatch); + } + DeclaratorSuffix::Array(dimension) => { + self.diagnostics.add_from_frontend_error( + ParseError::ExpectedBefore(TokenValue::LBracket, "function declarator"), + dimension.span, + ); + return Err(ParseProcessError::ErrorInMatch); + } + } + } + + match params { + Some(params) => Ok(params), + None => { + self.diagnostics.add_from_frontend_error( + ParseError::ExpectedBefore(TokenValue::LBrace, "function declarator"), + name_span, + ); + Err(ParseProcessError::ErrorInMatch) + } + } + } } diff --git a/src/ast/parser/stmt.rs b/src/ast/parser/stmt.rs index 5a31022..ac1474d 100644 --- a/src/ast/parser/stmt.rs +++ b/src/ast/parser/stmt.rs @@ -225,7 +225,7 @@ impl Parser { self.must_match_token(&TokenValue::LParen, "`(`")?; let init = if self.peek().value == TokenValue::Semicolon { None - } else if matches!(self.peek().value, TokenValue::TypeIdent(_)) { + } else if matches!(self.peek().value, TokenValue::Static | TokenValue::TypeIdent(_)) { Some(ForInit::VarDecl(self.parse_var_decl_stmt_inner(ParseType::MustParse, false)?)) } else { Some(ForInit::Expr(self.parse_expr()?)) diff --git a/src/lexer/lexer.rs b/src/lexer/lexer.rs index 72ce2ad..56c6efd 100644 --- a/src/lexer/lexer.rs +++ b/src/lexer/lexer.rs @@ -1,4 +1,4 @@ -use std::str::FromStr; +use std::{str::FromStr, sync::LazyLock}; use thiserror::Error; @@ -330,6 +330,17 @@ fn parse_puncuation( Ok(token_value) } +// static KEYWORDS: LazyLock> = LazyLock::new(|| { +// let mut m = HashMap::new(); +// m.insert("if", TokenValue::If); +// m.insert("else", TokenValue::Else); +// m.insert("while", TokenValue::While); +// m.insert("for", TokenValue::For); +// m.insert("return", TokenValue::Return); +// m.insert("break", TokenValue::Break); +// m.insert("continue", TokenValue::Continue); +// m +// }); fn parse_ident( str_iter: &mut Cursor, ) -> Result { @@ -370,6 +381,9 @@ fn parse_ident( if name.eq("continue") { return Ok(TokenValue::Continue); } + if name.eq("static") { + return Ok(TokenValue::Static); + } if let Some(type_ident) = TypeIdent::from_str(&name).ok() { return Ok(TokenValue::TypeIdent(type_ident)); } diff --git a/src/lexer/types.rs b/src/lexer/types.rs index 92bf94a..4cb66bc 100644 --- a/src/lexer/types.rs +++ b/src/lexer/types.rs @@ -25,7 +25,7 @@ pub enum TokenValue { LBracket, RBracket, Comma, Semicolon, - If, Else, While, For, Return, Break, Continue, + If, Else, While, For, Return, Break, Continue, Static, Eof, Unrecognized, @@ -84,6 +84,7 @@ impl std::fmt::Display for TokenValue { TokenValue::Return => write!(f, "return"), TokenValue::Break => write!(f, "break"), TokenValue::Continue => write!(f, "continue"), + TokenValue::Static => write!(f, "static"), TokenValue::Eof => write!(f, "end of input"), TokenValue::Unrecognized => write!(f, "unrecognized"), } @@ -104,7 +105,7 @@ pub enum TokenKind { LBracket, RBracket, Comma, Semicolon, - If, Else, While, For, Return, Break, Continue, + If, Else, While, For, Return, Break, Continue, Static, Eof, Unrecognized,