Added testing for base55, new identifier lexing

src/base_55.rs

@@ -122,3 +122,12 @@ pub fn num2char(number: i32) -> char {
         _ => ' ',
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    #[test] fn str_to_base55() {
+        let chrs: Vec<i32> = "AbleScript".chars().map(char2num).collect();
+        assert_eq!(chrs, &[-1, 2, 12, 5, -19, 3, 18, 9, 16, 20]);
+    }
+}

src/error.rs

@@ -10,6 +10,7 @@ pub struct Error {
 pub enum ErrorKind {
     SyntaxError(String),
     EndOfTokenStream,
+    InvalidIdentifier,
 }
 
 impl Error {

src/parser/mod.rs

@@ -68,7 +68,7 @@ impl<'a> Parser<'a> {
     ///
     /// `var [iden] = [literal];`
     fn variable_declaration(&mut self) -> Result<Expr, Error> {
-        let iden = self.require(Token::Identifier)?;
+        let iden = self.require_iden()?;
 
         let init = match self.lexer.next() {
             Some(Token::Semicolon) => None,
@@ -93,7 +93,7 @@ impl<'a> Parser<'a> {
     ///
     /// `functio [iden] ([expr], [expr]) { ... }
     fn function_declaration(&mut self) -> Result<Expr, Error> {
-        let iden = self.require(Token::Identifier)?;
+        let iden = self.require_iden()?;
         self.require(Token::LeftParenthesis)?;
         // TODO: Arguments
         self.require(Token::RightParenthesis)?;
@@ -110,7 +110,7 @@ impl<'a> Parser<'a> {
     /// `bff [iden] { ... }`
     fn bff_declaration(&mut self) -> Result<Expr, Error> {
         // TODO: Make it throw error when EOF
-        let iden = self.require(Token::Identifier)?;
+        let iden = self.require_iden()?;
         self.require(Token::LeftBrace)?;
         let mut code = String::new();
         while let Some(token) = self.lexer.next() {

src/parser/utils.rs

@@ -30,6 +30,17 @@ impl<'a> Parser<'a> {
         }
     }
 
+    pub(super) fn require_iden(&mut self) -> Result<String, Error> {
+        if let Some(Token::Identifier(id)) = self.lexer.next() {
+            Ok(id)
+        } else {
+            Err(Error {
+                kind: ErrorKind::InvalidIdentifier,
+                position: self.lexer.span(),
+            })
+        }
+    }
+
     pub(super) fn unexpected_token(&mut self, expected: Option<Token>) -> Error {
         Error {
             kind: ErrorKind::SyntaxError(format!(

src/tokens.rs

@@ -22,8 +22,8 @@ pub enum Token {
     Integer(i32),
 
     /// A C-complaint identifier
-    #[regex(r"[a-zA-Z_][a-zA-Z_0-9]*")]
+    #[regex(r"[a-zA-Z_][a-zA-Z_0-9]*", get_iden)]
-    Identifier,
+    Identifier(String),
 
     #[token("(")]
     LeftParenthesis,
@@ -134,7 +134,7 @@ fn get_int(lexer: &mut Lexer<Token>) -> Option<i32> {
 }
 
 fn get_string(lexer: &mut Lexer<Token>) -> String {
-    lexer.slice().to_owned()
+    lexer.slice().trim_matches('"').to_owned()
 }
 
 fn get_abool(lexer: &mut Lexer<Token>) -> Option<Abool> {
@@ -145,3 +145,43 @@ fn get_abool(lexer: &mut Lexer<Token>) -> Option<Abool> {
         _ => None,
     }
 }
+
+fn get_iden(lexer: &mut Lexer<Token>) -> String {
+    lexer.slice().to_owned()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Token;
+    use super::Token::*;
+    use logos::Logos;
+
+    #[test]
+    fn simple_fn() {
+        let code = "functio test() { var a = 3; if a == 3 { a print } }";
+        let expected = &[
+            Function,
+            Identifier("test".to_owned()),
+            LeftParenthesis,
+            RightParenthesis,
+            LeftBrace,
+            Variable,
+            Identifier("a".to_owned()),
+            Assignment,
+            Integer(3),
+            Semicolon,
+            If,
+            Identifier("a".to_owned()),
+            OpEq,
+            Integer(3),
+            LeftBrace,
+            Identifier("a".to_owned()),
+            Print,
+            RightBrace,
+            RightBrace,
+        ];
+        let lexer = Token::lexer(code);
+        let result: Vec<Token> = lexer.collect();
+        assert_eq!(result, expected);
+    }
+}