diff --git a/src/base_55.rs b/src/base_55.rs index 422eb1b..2efa1c3 100644 --- a/src/base_55.rs +++ b/src/base_55.rs @@ -122,3 +122,12 @@ pub fn num2char(number: i32) -> char { _ => ' ', } } + +#[cfg(test)] +mod tests { + use super::*; + #[test] fn str_to_base55() { + let chrs: Vec = "AbleScript".chars().map(char2num).collect(); + assert_eq!(chrs, &[-1, 2, 12, 5, -19, 3, 18, 9, 16, 20]); + } +} \ No newline at end of file diff --git a/src/error.rs b/src/error.rs index 1ae42e8..a5188eb 100644 --- a/src/error.rs +++ b/src/error.rs @@ -10,6 +10,7 @@ pub struct Error { pub enum ErrorKind { SyntaxError(String), EndOfTokenStream, + InvalidIdentifier, } impl Error { diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 5f7b08b..826b25c 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -68,7 +68,7 @@ impl<'a> Parser<'a> { /// /// `var [iden] = [literal];` fn variable_declaration(&mut self) -> Result { - let iden = self.require(Token::Identifier)?; + let iden = self.require_iden()?; let init = match self.lexer.next() { Some(Token::Semicolon) => None, @@ -93,7 +93,7 @@ impl<'a> Parser<'a> { /// /// `functio [iden] ([expr], [expr]) { ... } fn function_declaration(&mut self) -> Result { - let iden = self.require(Token::Identifier)?; + let iden = self.require_iden()?; self.require(Token::LeftParenthesis)?; // TODO: Arguments self.require(Token::RightParenthesis)?; @@ -110,7 +110,7 @@ impl<'a> Parser<'a> { /// `bff [iden] { ... }` fn bff_declaration(&mut self) -> Result { // TODO: Make it throw error when EOF - let iden = self.require(Token::Identifier)?; + let iden = self.require_iden()?; self.require(Token::LeftBrace)?; let mut code = String::new(); while let Some(token) = self.lexer.next() { diff --git a/src/parser/utils.rs b/src/parser/utils.rs index 218b2fa..90b6138 100644 --- a/src/parser/utils.rs +++ b/src/parser/utils.rs @@ -30,6 +30,17 @@ impl<'a> Parser<'a> { } } + pub(super) fn require_iden(&mut self) -> Result { + if let Some(Token::Identifier(id)) = self.lexer.next() { + Ok(id) + } else { + Err(Error { + kind: ErrorKind::InvalidIdentifier, + position: self.lexer.span(), + }) + } + } + pub(super) fn unexpected_token(&mut self, expected: Option) -> Error { Error { kind: ErrorKind::SyntaxError(format!( diff --git a/src/tokens.rs b/src/tokens.rs index fcc822a..6ee599a 100644 --- a/src/tokens.rs +++ b/src/tokens.rs @@ -22,8 +22,8 @@ pub enum Token { Integer(i32), /// A C-complaint identifier - #[regex(r"[a-zA-Z_][a-zA-Z_0-9]*")] - Identifier, + #[regex(r"[a-zA-Z_][a-zA-Z_0-9]*", get_iden)] + Identifier(String), #[token("(")] LeftParenthesis, @@ -134,7 +134,7 @@ fn get_int(lexer: &mut Lexer) -> Option { } fn get_string(lexer: &mut Lexer) -> String { - lexer.slice().to_owned() + lexer.slice().trim_matches('"').to_owned() } fn get_abool(lexer: &mut Lexer) -> Option { @@ -145,3 +145,43 @@ fn get_abool(lexer: &mut Lexer) -> Option { _ => None, } } + +fn get_iden(lexer: &mut Lexer) -> String { + lexer.slice().to_owned() +} + +#[cfg(test)] +mod tests { + use super::Token; + use super::Token::*; + use logos::Logos; + + #[test] + fn simple_fn() { + let code = "functio test() { var a = 3; if a == 3 { a print } }"; + let expected = &[ + Function, + Identifier("test".to_owned()), + LeftParenthesis, + RightParenthesis, + LeftBrace, + Variable, + Identifier("a".to_owned()), + Assignment, + Integer(3), + Semicolon, + If, + Identifier("a".to_owned()), + OpEq, + Integer(3), + LeftBrace, + Identifier("a".to_owned()), + Print, + RightBrace, + RightBrace, + ]; + let lexer = Token::lexer(code); + let result: Vec = lexer.collect(); + assert_eq!(result, expected); + } +} \ No newline at end of file