Continue work on parser, improved lexer

- Added literal parsing (improved lexing)
- Revised error handling

able-script-test/parse_test.able

@@ -1,10 +1,2 @@
-functio test() {
+var a = 3;
-    functio nested() {
+var b = 4;
-        var c = false;
-    }
-    var a = true;
-}
-
-functio another() {
-    var b = false;
-}

src/parser/item.rs

@@ -1,6 +1,18 @@
+use crate::variables::Value;
+
 #[derive(Debug, Clone)]
 pub enum Expr {
-    VariableDeclaration { iden: String, init: Option<String> },
+    VariableDeclaration {
-    FunctionDeclaration { iden: String, body: Vec<Expr> },
+        iden: String,
-    BfFDeclaration { iden: String, code: String },
+        init: Option<Box<Expr>>,
+    },
+    FunctionDeclaration {
+        iden: String,
+        body: Vec<Expr>,
+    },
+    BfFDeclaration {
+        iden: String,
+        code: String,
+    },
+    Literal(Value),
 }

src/parser/mod.rs

@@ -3,8 +3,11 @@ mod utils;
 
 use item::Expr;
 
-use crate::error::{Error, ErrorKind};
 use crate::tokens::Token;
+use crate::{
+    error::{Error, ErrorKind},
+    variables::Value,
+};
 
 use logos::Logos;
 
@@ -29,12 +32,13 @@ impl<'a> Parser<'a> {
             if token.is_none() {
                 return Ok(self.ast.clone());
             };
-            let expr = self.parse_expr(&token)?;
+
+            let expr = self.parse_expr(token)?;
             self.ast.push(expr);
         }
     }
 
-    fn parse_expr(&mut self, token: &Option<Token>) -> Result<Expr, Error> {
+    fn parse_expr(&mut self, token: Option<Token>) -> Result<Expr, Error> {
         if matches!(token, None) {
             return Err(Error {
                 kind: ErrorKind::EndOfTokenStream,
@@ -42,36 +46,23 @@ impl<'a> Parser<'a> {
             });
         }
 
-        Ok(todo!())
+        let token = token.unwrap();
-    }
+        let start = self.lexer.span().start;
 
-    /*
+        match token {
-    /// Start parsing Token Vector into Abstract Syntax Tree
+            Token::Variable => self.variable_declaration(),
-    pub fn parse(&mut self) -> Vec<Expr> {
+            Token::Function => self.function_declaration(),
-        let mut ast = vec![];
+            Token::BfFunction => self.bff_declaration(),
-        while let Some(token) = self.lexer.next() {
+            Token::String(x) => Ok(Expr::Literal(Value::Str(x))),
-            let expr = match token {
+            Token::Integer(x) => Ok(Expr::Literal(Value::Int(x))),
-                Token::Variable => self.variable_declaration(),
+            Token::Boolean(x) => Ok(Expr::Literal(Value::Bool(x))),
-                Token::Function => self.function_declaration(),
+            Token::Aboolean(x) => Ok(Expr::Literal(Value::Abool(x))),
-                Token::BfFunction => self.bff_declaration(),
+            _ => Err(Error {
-                Token::RightBrace => return ast,
+                kind: ErrorKind::SyntaxError("Unexpected identifier".to_owned()),
-                _ => Err(Error {
+                position: start..self.lexer.span().end,
-                    kind: ErrorKind::SyntaxError,
+            }),
-                    position: 0..0,
-                }),
-            };
-            match expr {
-                Ok(o) => ast.push(o),
-                Err(e) => {
-                    e.panic(self.lexer.slice());
-                    break;
-                }
-            }
         }
-
-        ast
     }
-    */
 
     /// Parse variable declaration
     ///
@@ -82,9 +73,10 @@ impl<'a> Parser<'a> {
         let init = match self.lexer.next() {
             Some(Token::Semicolon) => None,
             Some(Token::Assignment) => {
-                let value = self.require(Token::Boolean)?; // TODO: Shouldn't be limited to boolean (pattern match?)
+                let value = self.lexer.next();
+                let value = self.parse_expr(value)?; // TODO: Shouldn't be limited to boolean (pattern match?)
                 self.require(Token::Semicolon)?;
-                Some(value)
+                Some(Box::new(value))
             }
             _ => {
                 return Err(Error {
@@ -106,7 +98,9 @@ impl<'a> Parser<'a> {
         // TODO: Arguments
         self.require(Token::RightParenthesis)?;
         self.require(Token::LeftBrace)?;
-        let body = vec![];
+        let expr = self.lexer.next();
+        let expr = self.parse_expr(expr);
+        let body = vec![expr?];
 
         Ok(Expr::FunctionDeclaration { iden, body })
     }
@@ -115,6 +109,7 @@ impl<'a> Parser<'a> {
     ///
     /// `bff [iden] { ... }`
     fn bff_declaration(&mut self) -> Result<Expr, Error> {
+        // TODO: Make it throw error when EOF
         let iden = self.require(Token::Identifier)?;
         self.require(Token::LeftBrace)?;
         let mut code = String::new();
@@ -129,10 +124,9 @@ impl<'a> Parser<'a> {
                 | Token::LeftBracket
                 | Token::RightBracket => self.lexer.slice(),
                 Token::RightBrace => break,
-                _ => break,
+                _ => return Err(self.unexpected_token(None)),
             });
         }
-        self.require(Token::RightBrace)?;
         Ok(Expr::BfFDeclaration { iden, code })
     }
 }

src/parser/utils.rs

@@ -23,13 +23,21 @@ pub fn num2abool(number: i32) -> Abool {
 impl<'a> Parser<'a> {
     /// Require type of token as next and return it's value (sometimes irrelevant)
     pub(super) fn require(&mut self, with: Token) -> Result<String, Error> {
-        if self.lexer.next() == Some(with) {
+        if self.lexer.next() == Some(with.clone()) {
             Ok(self.lexer.slice().to_owned())
         } else {
-            Err(Error {
+            Err(self.unexpected_token(Some(with)))
-                kind: ErrorKind::SyntaxError("Mysterious parse error".to_owned()),
+        }
-                position: self.lexer.span(),
+    }
-            })
+
+    pub(super) fn unexpected_token(&mut self, expected: Option<Token>) -> Error {
+        Error {
+            kind: ErrorKind::SyntaxError(format!(
+                "Unexpected token: `{}` (required: `{:?}`)",
+                self.lexer.slice(),
+                expected
+            )),
+            position: self.lexer.span(),
         }
     }
 }

src/tokens.rs

@@ -1,23 +1,25 @@
-use logos::Logos;
+use logos::{Lexer, Logos};
 
-#[derive(Logos, Debug, PartialEq)]
+use crate::variables::Abool;
+
+#[derive(Logos, Debug, PartialEq, Clone)]
 pub enum Token {
     // Literals
     /// True, False
-    #[regex("true|false")]
+    #[regex("true|false", get_bool)]
-    Boolean,
+    Boolean(bool),
 
     /// Always, Sometimes, Never
-    #[regex("always|sometimes|never")]
+    #[regex("always|sometimes|never", get_abool)]
-    Aboolean,
+    Aboolean(Abool),
 
     /// String
-    #[regex("\"(\\.|[^\"])*\"")]
+    #[regex("\"(\\.|[^\"])*\"", get_string)]
-    String,
+    String(String),
 
     /// Integer
-    #[regex(r"[0-9]+")]
+    #[regex(r"[0-9]+", get_int)]
-    Integer,
+    Integer(i32),
 
     /// A C-complaint identifier
     #[regex(r"[a-zA-Z_][a-zA-Z_0-9]*")]
@@ -122,3 +124,24 @@ pub enum Token {
     #[error]
     Error,
 }
+
+fn get_bool(lexer: &mut Lexer<Token>) -> Option<bool> {
+    lexer.slice().parse().ok()
+}
+
+fn get_int(lexer: &mut Lexer<Token>) -> Option<i32> {
+    lexer.slice().parse().ok()
+}
+
+fn get_string(lexer: &mut Lexer<Token>) -> String {
+    lexer.slice().to_owned()
+}
+
+fn get_abool(lexer: &mut Lexer<Token>) -> Option<Abool> {
+    match lexer.slice() {
+        "always" => Some(Abool::Always),
+        "sometimes" => Some(Abool::Sometimes),
+        "never" => Some(Abool::Never),
+        _ => None,
+    }
+}