Continue work on parser, improved lexer

- Added literal parsing (improved lexing)
- Revised error handling
This commit is contained in:
Erin 2021-04-27 10:51:39 +02:00 committed by ondra05
parent 47400ee2ce
commit 83c549607c
5 changed files with 92 additions and 63 deletions

View file

@ -1,10 +1,2 @@
functio test() { var a = 3;
functio nested() { var b = 4;
var c = false;
}
var a = true;
}
functio another() {
var b = false;
}

View file

@ -1,6 +1,18 @@
use crate::variables::Value;
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub enum Expr { pub enum Expr {
VariableDeclaration { iden: String, init: Option<String> }, VariableDeclaration {
FunctionDeclaration { iden: String, body: Vec<Expr> }, iden: String,
BfFDeclaration { iden: String, code: String }, init: Option<Box<Expr>>,
},
FunctionDeclaration {
iden: String,
body: Vec<Expr>,
},
BfFDeclaration {
iden: String,
code: String,
},
Literal(Value),
} }

View file

@ -3,8 +3,11 @@ mod utils;
use item::Expr; use item::Expr;
use crate::error::{Error, ErrorKind};
use crate::tokens::Token; use crate::tokens::Token;
use crate::{
error::{Error, ErrorKind},
variables::Value,
};
use logos::Logos; use logos::Logos;
@ -29,12 +32,13 @@ impl<'a> Parser<'a> {
if token.is_none() { if token.is_none() {
return Ok(self.ast.clone()); return Ok(self.ast.clone());
}; };
let expr = self.parse_expr(&token)?;
let expr = self.parse_expr(token)?;
self.ast.push(expr); self.ast.push(expr);
} }
} }
fn parse_expr(&mut self, token: &Option<Token>) -> Result<Expr, Error> { fn parse_expr(&mut self, token: Option<Token>) -> Result<Expr, Error> {
if matches!(token, None) { if matches!(token, None) {
return Err(Error { return Err(Error {
kind: ErrorKind::EndOfTokenStream, kind: ErrorKind::EndOfTokenStream,
@ -42,36 +46,23 @@ impl<'a> Parser<'a> {
}); });
} }
Ok(todo!()) let token = token.unwrap();
} let start = self.lexer.span().start;
/* match token {
/// Start parsing Token Vector into Abstract Syntax Tree Token::Variable => self.variable_declaration(),
pub fn parse(&mut self) -> Vec<Expr> { Token::Function => self.function_declaration(),
let mut ast = vec![]; Token::BfFunction => self.bff_declaration(),
while let Some(token) = self.lexer.next() { Token::String(x) => Ok(Expr::Literal(Value::Str(x))),
let expr = match token { Token::Integer(x) => Ok(Expr::Literal(Value::Int(x))),
Token::Variable => self.variable_declaration(), Token::Boolean(x) => Ok(Expr::Literal(Value::Bool(x))),
Token::Function => self.function_declaration(), Token::Aboolean(x) => Ok(Expr::Literal(Value::Abool(x))),
Token::BfFunction => self.bff_declaration(), _ => Err(Error {
Token::RightBrace => return ast, kind: ErrorKind::SyntaxError("Unexpected identifier".to_owned()),
_ => Err(Error { position: start..self.lexer.span().end,
kind: ErrorKind::SyntaxError, }),
position: 0..0,
}),
};
match expr {
Ok(o) => ast.push(o),
Err(e) => {
e.panic(self.lexer.slice());
break;
}
}
} }
ast
} }
*/
/// Parse variable declaration /// Parse variable declaration
/// ///
@ -82,9 +73,10 @@ impl<'a> Parser<'a> {
let init = match self.lexer.next() { let init = match self.lexer.next() {
Some(Token::Semicolon) => None, Some(Token::Semicolon) => None,
Some(Token::Assignment) => { Some(Token::Assignment) => {
let value = self.require(Token::Boolean)?; // TODO: Shouldn't be limited to boolean (pattern match?) let value = self.lexer.next();
let value = self.parse_expr(value)?; // TODO: Shouldn't be limited to boolean (pattern match?)
self.require(Token::Semicolon)?; self.require(Token::Semicolon)?;
Some(value) Some(Box::new(value))
} }
_ => { _ => {
return Err(Error { return Err(Error {
@ -106,7 +98,9 @@ impl<'a> Parser<'a> {
// TODO: Arguments // TODO: Arguments
self.require(Token::RightParenthesis)?; self.require(Token::RightParenthesis)?;
self.require(Token::LeftBrace)?; self.require(Token::LeftBrace)?;
let body = vec![]; let expr = self.lexer.next();
let expr = self.parse_expr(expr);
let body = vec![expr?];
Ok(Expr::FunctionDeclaration { iden, body }) Ok(Expr::FunctionDeclaration { iden, body })
} }
@ -115,6 +109,7 @@ impl<'a> Parser<'a> {
/// ///
/// `bff [iden] { ... }` /// `bff [iden] { ... }`
fn bff_declaration(&mut self) -> Result<Expr, Error> { fn bff_declaration(&mut self) -> Result<Expr, Error> {
// TODO: Make it throw error when EOF
let iden = self.require(Token::Identifier)?; let iden = self.require(Token::Identifier)?;
self.require(Token::LeftBrace)?; self.require(Token::LeftBrace)?;
let mut code = String::new(); let mut code = String::new();
@ -129,10 +124,9 @@ impl<'a> Parser<'a> {
| Token::LeftBracket | Token::LeftBracket
| Token::RightBracket => self.lexer.slice(), | Token::RightBracket => self.lexer.slice(),
Token::RightBrace => break, Token::RightBrace => break,
_ => break, _ => return Err(self.unexpected_token(None)),
}); });
} }
self.require(Token::RightBrace)?;
Ok(Expr::BfFDeclaration { iden, code }) Ok(Expr::BfFDeclaration { iden, code })
} }
} }

View file

@ -23,13 +23,21 @@ pub fn num2abool(number: i32) -> Abool {
impl<'a> Parser<'a> { impl<'a> Parser<'a> {
/// Require type of token as next and return it's value (sometimes irrelevant) /// Require type of token as next and return it's value (sometimes irrelevant)
pub(super) fn require(&mut self, with: Token) -> Result<String, Error> { pub(super) fn require(&mut self, with: Token) -> Result<String, Error> {
if self.lexer.next() == Some(with) { if self.lexer.next() == Some(with.clone()) {
Ok(self.lexer.slice().to_owned()) Ok(self.lexer.slice().to_owned())
} else { } else {
Err(Error { Err(self.unexpected_token(Some(with)))
kind: ErrorKind::SyntaxError("Mysterious parse error".to_owned()), }
position: self.lexer.span(), }
})
pub(super) fn unexpected_token(&mut self, expected: Option<Token>) -> Error {
Error {
kind: ErrorKind::SyntaxError(format!(
"Unexpected token: `{}` (required: `{:?}`)",
self.lexer.slice(),
expected
)),
position: self.lexer.span(),
} }
} }
} }

View file

@ -1,23 +1,25 @@
use logos::Logos; use logos::{Lexer, Logos};
#[derive(Logos, Debug, PartialEq)] use crate::variables::Abool;
#[derive(Logos, Debug, PartialEq, Clone)]
pub enum Token { pub enum Token {
// Literals // Literals
/// True, False /// True, False
#[regex("true|false")] #[regex("true|false", get_bool)]
Boolean, Boolean(bool),
/// Always, Sometimes, Never /// Always, Sometimes, Never
#[regex("always|sometimes|never")] #[regex("always|sometimes|never", get_abool)]
Aboolean, Aboolean(Abool),
/// String /// String
#[regex("\"(\\.|[^\"])*\"")] #[regex("\"(\\.|[^\"])*\"", get_string)]
String, String(String),
/// Integer /// Integer
#[regex(r"[0-9]+")] #[regex(r"[0-9]+", get_int)]
Integer, Integer(i32),
/// A C-complaint identifier /// A C-complaint identifier
#[regex(r"[a-zA-Z_][a-zA-Z_0-9]*")] #[regex(r"[a-zA-Z_][a-zA-Z_0-9]*")]
@ -122,3 +124,24 @@ pub enum Token {
#[error] #[error]
Error, Error,
} }
fn get_bool(lexer: &mut Lexer<Token>) -> Option<bool> {
lexer.slice().parse().ok()
}
fn get_int(lexer: &mut Lexer<Token>) -> Option<i32> {
lexer.slice().parse().ok()
}
fn get_string(lexer: &mut Lexer<Token>) -> String {
lexer.slice().to_owned()
}
fn get_abool(lexer: &mut Lexer<Token>) -> Option<Abool> {
match lexer.slice() {
"always" => Some(Abool::Always),
"sometimes" => Some(Abool::Sometimes),
"never" => Some(Abool::Never),
_ => None,
}
}