Divided token types

- Item = Expr | Stmt
- Added REPL (AST dump)
- Removed operator parsing because it was horrible and I need to redo it
This commit is contained in:
Erin 2021-05-01 13:44:58 +02:00 committed by ondra05
parent c26c4a14bb
commit d7cf1b013d
9 changed files with 268 additions and 75 deletions

156
Cargo.lock generated
View file

@ -9,6 +9,7 @@ dependencies = [
"clap",
"logos",
"rand",
"rustyline",
]
[[package]]
@ -43,6 +44,12 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
[[package]]
name = "cc"
version = "1.0.67"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd"
[[package]]
name = "cfg-if"
version = "1.0.0"
@ -64,12 +71,49 @@ dependencies = [
"vec_map",
]
[[package]]
name = "dirs-next"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
dependencies = [
"cfg-if",
"dirs-sys-next",
]
[[package]]
name = "dirs-sys-next"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"
dependencies = [
"libc",
"redox_users",
"winapi",
]
[[package]]
name = "endian-type"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d"
[[package]]
name = "fnv"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "fs2"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "getrandom"
version = "0.2.2"
@ -96,6 +140,15 @@ version = "0.2.93"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9385f66bf6105b241aa65a61cb923ef20efc665cb9f9bb50ac2f0c4b7f378d41"
[[package]]
name = "log"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
dependencies = [
"cfg-if",
]
[[package]]
name = "logos"
version = "0.12.0"
@ -120,6 +173,33 @@ dependencies = [
"utf8-ranges",
]
[[package]]
name = "memchr"
version = "2.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
[[package]]
name = "nibble_vec"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43"
dependencies = [
"smallvec",
]
[[package]]
name = "nix"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa9b4819da1bc61c0ea48b63b7bc8604064dd43013e7cc325df098d49cd7c18a"
dependencies = [
"bitflags",
"cc",
"cfg-if",
"libc",
]
[[package]]
name = "ppv-lite86"
version = "0.2.10"
@ -144,6 +224,16 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "radix_trie"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd"
dependencies = [
"endian-type",
"nibble_vec",
]
[[package]]
name = "rand"
version = "0.8.3"
@ -184,12 +274,66 @@ dependencies = [
"rand_core",
]
[[package]]
name = "redox_syscall"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85dd92e586f7355c633911e11f77f3d12f04b1b1bd76a198bd34ae3af8341ef2"
dependencies = [
"bitflags",
]
[[package]]
name = "redox_users"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64"
dependencies = [
"getrandom",
"redox_syscall",
]
[[package]]
name = "regex-syntax"
version = "0.6.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548"
[[package]]
name = "rustyline"
version = "8.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e1b597fcd1eeb1d6b25b493538e5aa19629eb08932184b85fef931ba87e893"
dependencies = [
"bitflags",
"cfg-if",
"dirs-next",
"fs2",
"libc",
"log",
"memchr",
"nix",
"radix_trie",
"scopeguard",
"smallvec",
"unicode-segmentation",
"unicode-width",
"utf8parse",
"winapi",
]
[[package]]
name = "scopeguard"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "smallvec"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
[[package]]
name = "strsim"
version = "0.8.0"
@ -216,6 +360,12 @@ dependencies = [
"unicode-width",
]
[[package]]
name = "unicode-segmentation"
version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796"
[[package]]
name = "unicode-width"
version = "0.1.8"
@ -234,6 +384,12 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4ae116fef2b7fea257ed6440d3cfcff7f190865f170cdad00bb6465bf18ecba"
[[package]]
name = "utf8parse"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372"
[[package]]
name = "vec_map"
version = "0.8.2"

View file

@ -10,3 +10,4 @@ edition = "2018"
clap="*"
logos = "0.12"
rand = "*"
rustyline = "8.0.0"

View file

@ -1 +0,0 @@
1 + 4

View file

@ -4,6 +4,7 @@ mod base_55;
mod error;
mod lexer;
mod parser;
mod repl;
mod variables;
use clap::{App, Arg};
@ -14,7 +15,7 @@ fn main() {
let matches = App::new("AbleScript")
.version(env!("CARGO_PKG_VERSION"))
.author("Able <abl3theabove@gmail.com>")
.about("Does awesome things")
.about("AbleScript interpreter")
.arg(
Arg::with_name("file")
.short("f")
@ -36,8 +37,8 @@ fn main() {
println!("{:#?}", ast);
}
None => {
println!("hi");
//start the prompt
println!("Hi [AbleScript {}] - AST Printer", env!("CARGO_PKG_VERSION"));
repl::repl();
}
}
}

View file

@ -1,47 +1,61 @@
use std::collections::HashMap;
use crate::variables::Value;
#[derive(Debug, Clone, Hash, PartialEq, Eq)]
pub struct Iden(pub String);
impl From<Iden> for Expr {
fn from(iden: Iden) -> Self {
Self::Identifier(iden)
#[derive(Debug, Clone)]
pub enum Item {
Expr(Expr),
Stmt(Stmt),
}
impl From<Expr> for Item {
fn from(e: Expr) -> Self {
Item::Expr(e)
}
}
impl From<Stmt> for Item {
fn from(s: Stmt) -> Self {
Item::Stmt(s)
}
}
#[derive(Debug, Clone)]
pub enum Expr {
Add { left: Box<Expr>, right: Box<Expr> },
Subtract { left: Box<Expr>, right: Box<Expr> },
Multiply { left: Box<Expr>, right: Box<Expr> },
Divide { left: Box<Expr>, right: Box<Expr> },
FunctionCall { iden: Iden, args: Vec<Expr> },
Literal(Value),
Identifier(Iden),
}
impl From<Iden> for Expr {
fn from(i: Iden) -> Self {
Self::Identifier(i)
}
}
#[derive(Debug, Clone)]
pub enum Stmt {
VariableDeclaration {
iden: String,
init: Option<Box<Expr>>,
iden: Iden,
init: Option<Box<Item>>,
},
FunctionDeclaration {
iden: String,
iden: Iden,
args: Vec<Iden>,
body: Vec<Expr>,
body: Vec<Item>,
},
BfFDeclaration {
iden: String,
iden: Iden,
body: String,
},
If {
cond: Box<Expr>,
body: Vec<Expr>,
cond: Box<Item>,
body: Vec<Item>,
},
FunctionCall {
iden: Iden,
args: Vec<Expr>,
},
Addition {
left: Box<Expr>,
right: Box<Expr>,
},
Literal(Value),
Identifier(Iden),
Melo(Iden),
}

View file

@ -2,19 +2,22 @@ mod item;
mod ops;
mod utils;
use item::Expr;
use item::Item;
use crate::{
error::{Error, ErrorKind},
lexer::PeekableLexer,
parser::item::{Expr, Stmt},
variables::Value,
};
use crate::{lexer::Token, parser::item::Iden};
pub type ParseResult = Result<Item, Error>;
/// Parser structure / state machine
pub struct Parser<'a> {
lexer: PeekableLexer<'a>,
ast: Vec<Expr>,
ast: Vec<Item>,
}
impl<'a> Parser<'a> {
@ -26,19 +29,19 @@ impl<'a> Parser<'a> {
}
}
pub fn init(&mut self) -> Result<Vec<Expr>, Error> {
pub fn init(&mut self) -> Result<Vec<Item>, Error> {
loop {
let token = self.lexer.next();
if token.is_none() {
return Ok(self.ast.clone());
};
let expr = self.parse_expr(token)?;
let expr = self.parse_item(token)?;
self.ast.push(expr);
}
}
fn parse_expr(&mut self, token: Option<Token>) -> Result<Expr, Error> {
fn parse_item(&mut self, token: Option<Token>) -> ParseResult {
if matches!(token, None) {
return Err(Error {
kind: ErrorKind::EndOfTokenStream,
@ -50,7 +53,7 @@ impl<'a> Parser<'a> {
let start = self.lexer.span().start;
match token {
Token::Identifier(_) => self.parse_ops(token),
Token::Identifier(_) => self.parse_ops(token).map(|x| x.into()),
// Control flow
Token::If => self.if_cond(),
@ -60,17 +63,17 @@ impl<'a> Parser<'a> {
Token::BfFunction => self.bff_declaration(),
// Literals
Token::String(x) => Ok(Expr::Literal(Value::Str(x))),
Token::Integer(x) => Ok(Expr::Literal(Value::Int(x))),
Token::Boolean(x) => Ok(Expr::Literal(Value::Bool(x))),
Token::Aboolean(x) => Ok(Expr::Literal(Value::Abool(x))),
Token::String(x) => Ok(Expr::Literal(Value::Str(x)).into()),
Token::Integer(x) => Ok(Expr::Literal(Value::Int(x)).into()),
Token::Boolean(x) => Ok(Expr::Literal(Value::Bool(x)).into()),
Token::Aboolean(x) => Ok(Expr::Literal(Value::Abool(x)).into()),
// Prefix keywords
// Melo - ban variable from next usage (runtime error)
Token::Melo => {
let e = self.require_iden()?;
self.require(Token::Semicolon)?;
Ok(Expr::Melo(Iden(e)))
Ok(Stmt::Melo(e).into())
}
_ => Err(Error {
@ -83,14 +86,14 @@ impl<'a> Parser<'a> {
/// Parse variable declaration
///
/// `var [iden] = [literal];`
fn variable_declaration(&mut self) -> Result<Expr, Error> {
fn variable_declaration(&mut self) -> ParseResult {
let iden = self.require_iden()?;
let init = match self.lexer.next() {
Some(Token::Semicolon) => None,
Some(Token::Assignment) => {
let value = self.lexer.next();
let value = self.parse_expr(value)?;
let value = self.parse_item(value)?;
self.require(Token::Semicolon)?;
Some(Box::new(value))
}
@ -102,13 +105,13 @@ impl<'a> Parser<'a> {
}
};
Ok(Expr::VariableDeclaration { iden, init })
Ok(Stmt::VariableDeclaration { iden, init }.into())
}
/// Declare function
///
/// `functio [iden] ([expr], [expr]) { ... }
fn function_declaration(&mut self) -> Result<Expr, Error> {
fn function_declaration(&mut self) -> ParseResult {
let iden = self.require_iden()?;
self.require(Token::LeftParenthesis)?;
let args = vec![];
@ -118,13 +121,13 @@ impl<'a> Parser<'a> {
// Parse function body
let body = self.parse_body()?;
Ok(Expr::FunctionDeclaration { iden, args, body })
Ok(Stmt::FunctionDeclaration { iden, args, body }.into())
}
/// Declare BF FFI Function
///
/// `bff [iden] { ... }`
fn bff_declaration(&mut self) -> Result<Expr, Error> {
fn bff_declaration(&mut self) -> ParseResult {
let iden = self.require_iden()?;
self.require(Token::LeftBrace)?;
@ -155,23 +158,24 @@ impl<'a> Parser<'a> {
_ => return Err(self.unexpected_token(None)),
});
}
Ok(Expr::BfFDeclaration { iden, body })
Ok(Stmt::BfFDeclaration { iden, body }.into())
}
/// Parse If-expression
pub fn if_cond(&mut self) -> Result<Expr, Error> {
pub fn if_cond(&mut self) -> ParseResult {
self.require(Token::LeftParenthesis)?;
let cond = self.lexer.next();
let cond = self.parse_expr(cond)?;
let cond = self.parse_item(cond)?;
self.require(Token::RightParenthesis)?;
self.require(Token::LeftBrace)?;
let body = self.parse_body()?;
Ok(Expr::If {
Ok(Stmt::If {
cond: Box::new(cond),
body,
})
}
.into())
}
}

View file

@ -1,39 +1,27 @@
use super::*;
type ExprResult = Result<Expr, Error>;
impl<'a> Parser<'a> {
/// Parse operations (got identifier/value)
pub(super) fn parse_ops(&mut self, token: Token) -> Result<Expr, Error> {
let iden = if let Token::Identifier(i) = token {
Iden(i)
} else {
unimplemented!()
};
let mut buf = Vec::new();
buf.push(match self.lexer.peek() {
Some(Token::LeftParenthesis) => self.fn_call(iden)?,
_ => unimplemented!(),
});
Ok(buf[0].clone())
pub(super) fn parse_ops(&mut self, token: Token) -> ExprResult {
todo!()
}
/// Parse function call
fn fn_call(&mut self, iden: Iden) -> Result<Expr, Error> {
fn fn_call(&mut self, iden: Iden) -> ExprResult {
self.lexer.next();
let mut args: Vec<Expr> = Vec::new();
while let Some(token) = self.lexer.peek() {
match token {
Token::Identifier(id) => {
args.push(Iden(id.clone()).into());
args.push(Expr::Identifier(Iden(id.clone())));
self.lexer.next();
}
Token::RightParenthesis => break,
_ => {
let next = self.lexer.next();
args.push(self.parse_expr(next)?)
}
}
self.require(Token::Comma)?;

View file

@ -2,7 +2,7 @@ use crate::error::{Error, ErrorKind};
use crate::lexer::Token;
use crate::variables::Abool;
use super::{item::Expr, Parser};
use super::{Parser, item::{Iden, Item}};
pub fn abool2num(abool: Abool) -> i32 {
match abool {
@ -31,9 +31,9 @@ impl<'a> Parser<'a> {
}
/// Require an identifier on next and return it
pub(super) fn require_iden(&mut self) -> Result<String, Error> {
pub(super) fn require_iden(&mut self) -> Result<Iden, Error> {
if let Some(Token::Identifier(id)) = self.lexer.next() {
Ok(id)
Ok(Iden(id))
} else {
Err(Error {
kind: ErrorKind::InvalidIdentifier,
@ -58,7 +58,7 @@ impl<'a> Parser<'a> {
}
}
pub(super) fn parse_body(&mut self) -> Result<Vec<Expr>, Error> {
pub(super) fn parse_body(&mut self) -> Result<Vec<Item>, Error> {
let mut body = Vec::new();
loop {
let token = {
@ -76,7 +76,7 @@ impl<'a> Parser<'a> {
if token == Token::RightBrace {
break;
}
body.push(self.parse_expr(Some(token))?);
body.push(self.parse_item(Some(token))?);
}
Ok(body)
}

30
src/repl.rs Normal file
View file

@ -0,0 +1,30 @@
use rustyline::Editor;
use crate::parser::Parser;
pub fn repl() {
let mut rl = Editor::<()>::new();
loop {
let readline = rl.readline(":: ");
match readline {
Ok(line) => {
if &line == "exit" {
println!("bye");
break;
}
let mut parser = Parser::new(&line);
let ast = parser.init();
println!("{:#?}", ast);
}
Err(rustyline::error::ReadlineError::Eof) => {
println!("bye");
break;
}
Err(rustyline::error::ReadlineError::Interrupted) => (),
Err(e) => {
println!("Error: {:?}", e);
break;
}
}
}
}