1
1
Fork 0
mirror of https://github.com/azur1s/bobbylisp.git synced 2024-10-16 02:37:40 -05:00

feat: lexer & overall rework

This commit is contained in:
Natapat Samutpong 2022-02-12 13:28:53 +07:00
parent 4f97a39f86
commit 24daf588b0
15 changed files with 241 additions and 477 deletions

32
Cargo.lock generated
View file

@ -76,6 +76,14 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "hycron"
version = "0.1.0"
dependencies = [
"clap",
"nom",
]
[[package]] [[package]]
name = "indexmap" name = "indexmap"
version = "1.8.0" version = "1.8.0"
@ -104,6 +112,23 @@ version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
[[package]]
name = "minimal-lexical"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
[[package]]
name = "nom"
version = "7.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b1d11e1ef389c76fe5b81bcaf2ea32cf88b62bc494e19f493d0b30e7a930109"
dependencies = [
"memchr",
"minimal-lexical",
"version_check",
]
[[package]] [[package]]
name = "os_str_bytes" name = "os_str_bytes"
version = "6.0.0" version = "6.0.0"
@ -199,13 +224,6 @@ version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
name = "vyc"
version = "0.1.0"
dependencies = [
"clap",
]
[[package]] [[package]]
name = "winapi" name = "winapi"
version = "0.3.9" version = "0.3.9"

View file

@ -1,5 +1,5 @@
[package] [package]
name = "vyc" name = "hycron"
version = "0.1.0" version = "0.1.0"
edition = "2021" edition = "2021"
@ -7,3 +7,4 @@ edition = "2021"
[dependencies] [dependencies]
clap = { version = "3.0.14", features = ["derive"] } clap = { version = "3.0.14", features = ["derive"] }
nom = "7.1.0"

View file

@ -1 +0,0 @@
(def number 1)

3
example/hello_world.hyc Normal file
View file

@ -0,0 +1,3 @@
let msg :: String = "Hello, World";
func add2 :: (a: Int, b: Int) -> Int = a + b;
func main :: () = puts (msg);

View file

@ -1,2 +0,0 @@
(def message "Hello, World")
(print message)

View file

@ -1 +0,0 @@
(if (equal 1 1) (print "a"))

View file

@ -3,7 +3,7 @@ use clap::{ Parser, Subcommand };
const VERSION: &str = env!("CARGO_PKG_VERSION"); const VERSION: &str = env!("CARGO_PKG_VERSION");
/// Vy language compiler. /// Hycron language compiler.
#[derive(Parser, Debug)] #[derive(Parser, Debug)]
#[clap( #[clap(
version = VERSION, version = VERSION,

31
src/front/helper.rs Normal file
View file

@ -0,0 +1,31 @@
use std::str::{self, Utf8Error, FromStr};
pub type Bytes = [u8];
#[macro_export]
macro_rules! syntax {
($func_name: ident, $tag_string: literal, $output_token: expr) => {
fn $func_name<'a>(s: &'a Bytes) -> IResult<&Bytes, Token> {
map(tag($tag_string), |_| $output_token)(s)
}
};
}
pub fn concat_slice_vec(a: &Bytes, b: Vec<u8>) -> Vec<u8> {
let mut result = a.to_vec();
result.extend(&b);
result
}
pub fn convert_vec_utf8(v: Vec<u8>) -> Result<String, Utf8Error> {
let slice = v.as_slice();
str::from_utf8(slice).map(|s| s.to_owned())
}
pub fn str_from_bytes(c: &Bytes) -> Result<&str, Utf8Error> {
str::from_utf8(c)
}
pub fn str_to_from_str<F: FromStr>(c: &str) -> Result<F, F::Err> {
FromStr::from_str(c)
}

137
src/front/lex.rs Normal file
View file

@ -0,0 +1,137 @@
use nom::{
branch::alt,
bytes::complete::{tag, take},
character::complete::{multispace0, alphanumeric1, alpha1, digit1},
combinator::{map, map_res, recognize},
IResult,
multi::many0,
sequence::{delimited, pair}, AsBytes,
};
use crate::syntax;
use super::{
model::Token,
helper::{Bytes, convert_vec_utf8, concat_slice_vec, str_from_bytes, str_to_from_str},
};
// Comparison
syntax! { equal_operator , "==", Token::Eq }
syntax! { not_equal_operator , "!=", Token::NEq }
syntax! { less_than_operator , "<" , Token::Lt }
syntax! { greater_than_operator , ">" , Token::Gt }
syntax! { less_than_equal_operator , "<=", Token::Lte }
syntax! { greater_than_equal_operator , ">=", Token::Gte }
// Arithmetic
syntax! { assign_operator , "=", Token::Assign }
syntax! { add_operator , "+", Token::Plus }
syntax! { subtract_operator , "-", Token::Minus }
syntax! { multiply_operator , "*", Token::Mul }
syntax! { divide_operator , "/", Token::Div }
syntax! { not_operator , "!", Token::Not }
// Punctuations
syntax! { typehint_punctuation , "::", Token::Typehint }
syntax! { lparen_punctuation , "(", Token::LParen }
syntax! { rparen_punctuation , ")", Token::RParen }
syntax! { semicolon_punctuation , ";", Token::Semicolon }
syntax! { colon_punctuation , ":", Token::Colon }
// Operator & Punctuation
fn lex_operator_punctuation(input: &Bytes) -> IResult<&Bytes, Token> {
alt((
equal_operator, not_equal_operator,
less_than_operator, greater_than_operator,
less_than_equal_operator, greater_than_equal_operator,
assign_operator,
add_operator, subtract_operator, multiply_operator, divide_operator,
not_operator,
typehint_punctuation,
lparen_punctuation, rparen_punctuation,
semicolon_punctuation, colon_punctuation,
))(input)
}
// String
fn string_value(input: &Bytes) -> IResult<&Bytes, Vec<u8>> {
let (i1, c1) = take(1usize)(input)?;
match c1.as_bytes() {
b"\"" => Ok((input, vec![])),
b"\\" => {
let (i2, c2) = take(1usize)(i1)?;
string_value(i2).map(|(slice, done)| (slice, concat_slice_vec(c2, done)))
}
c => string_value(i1).map(|(slice, done)| (slice, concat_slice_vec(c, done)))
}
}
fn string(input: &Bytes) -> IResult<&Bytes, String> {
delimited(tag("\""), map_res(string_value, convert_vec_utf8), tag("\""))(input)
}
fn lex_string(input: &Bytes) -> IResult<&Bytes, Token> {
map(string, |s| Token::String(s))(input)
}
// Reserved keywords & Identifiers
fn lex_reserved_identifier(input: &Bytes) -> IResult<&Bytes, Token> {
map_res(
recognize(pair(
alt((alpha1, tag("_"))
),
many0(alt((alphanumeric1, tag("_")))),
)),
|s| {
let c = str_from_bytes(s);
c.map(|syntax| match syntax {
"if" => Token::If,
"else" => Token::Else,
"let" => Token::Let,
"func" => Token::Func,
"true" => Token::Bool(true),
"false" => Token::Bool(false),
_ => Token::Identifier(syntax.to_string()),
})
},
)(input)
}
// Integers
fn lex_integer(input: &Bytes) -> IResult<&Bytes, Token> {
map(
map_res(
map_res(digit1, str_from_bytes),
str_to_from_str,
),
Token::Int,
)(input)
}
// Illegal tokens
fn lex_illegal(input: &Bytes) -> IResult<&Bytes, Token> {
map(take(1usize), |_| Token::Illegal)(input)
}
// Tokens
fn lex_token(input: &Bytes) -> IResult<&Bytes, Token> {
alt((
lex_operator_punctuation,
lex_string,
lex_reserved_identifier,
lex_integer,
lex_illegal,
))(input)
}
fn lex_tokens(input: &Bytes) -> IResult<&Bytes, Vec<Token>> {
many0(delimited(multispace0, lex_token, multispace0))(input)
}
pub struct Lexer;
impl Lexer {
pub fn lex_tokens(input: &Bytes) -> IResult<&Bytes, Vec<Token>> {
lex_tokens(input).map(|(slice, result)| (slice, [&result[..], &vec![Token::EndOfFile][..]].concat()))
}
}

View file

@ -1 +1,4 @@
pub mod parser; pub mod model;
pub mod helper;
pub mod lex;

29
src/front/model.rs Normal file
View file

@ -0,0 +1,29 @@
#[derive(Clone, Debug, PartialEq)]
pub enum Token {
Illegal, EndOfFile,
Identifier(String), String(String),
Int(i64), Bool(bool),
Assign, Typehint,
Plus, Minus, Mul, Div, Not,
Eq, NEq, Lt, Gt, Lte, Gte,
LParen, RParen, Semicolon, Colon,
If, Else, Let, Func,
}
/// Token struct with position information.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Tokens<'a> {
pub tokens: &'a [Token],
pub start: usize, pub end: usize,
}
impl<'a> Tokens<'a> {
pub fn new(tokens: &'a [Token]) -> Self {
Tokens { tokens, start: 0, end: tokens.len(), }
}
}

View file

@ -1,314 +0,0 @@
// This implementation of parser is heavily inspired by
// brundonsmith/rust_lisp (https://github.com/brundonsmith/rust_lisp)
// go check them out!
use std::{ rc::Rc, fmt };
#[derive(Debug, Clone)]
pub enum Value {
// Boolean types
True, False,
// Numbers
Int(i64), Float(f64),
String(String), Symbol(String),
List(Rc<Value>, Rc<Vec<Value>>),
Nil,
}
impl fmt::Display for Value {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
Value::True => write!(f, "$True"),
Value::False => write!(f, "$False"),
Value::Int(i) => write!(f, "${}", i),
Value::Float(fl) => write!(f, "${}", fl),
Value::String(s) => write!(f, "$\"{}\"", s),
Value::Symbol(s) => write!(f, "${}", s),
Value::List(car, cdr) => {
write!(f, "(")?;
write!(f, "{}", car)?;
for item in cdr.iter().cloned() { write!(f, " {}", item)?; }
write!(f, ")")
},
Value::Nil => write!(f, "nil"),
}
}
}
#[derive(Debug, Clone)]
pub enum Tree {
Atom { atom: Value, quote: bool },
List { vec: Vec<Tree>, quote: bool },
}
impl Tree {
fn into_expr(self) -> Value {
match self {
Tree::Atom { atom, quote } => {
if quote {
Value::List(
Rc::new(Value::Symbol(String::from("quote"))),
Rc::new(vec![atom])
)
} else {
atom
}
},
Tree::List { vec, quote } => {
let list = Value::List(
Rc::new(vec[0].clone().into_expr()),
Rc::new(vec[1..].iter().map(|a| a.clone().into_expr()).collect())
);
if quote {
Value::List(
Rc::new(Value::Symbol(String::from("quote"))),
Rc::new(vec![list])
)
} else {
list
}
}
}
}
}
// --- Start tokenizer ---
const SPECIAL_CHARS: [&str; 4] = ["(", ")", "'", "..."];
/// Match the characters from `with` with the characters from `from`
/// Example: match_front("123", "12") -> true
fn match_front(from: &str, with: &str) -> bool { with.chars().zip(from.chars()).all(|(a, b)| a == b) }
/// Get length from `from` until `predicate`
/// Example: match_pred("abcdef", |c| c != 'f') -> Some(5)
fn match_predicate<F: Fn(char) -> bool>(from: &str, predicate: F) -> Option<usize> {
from.char_indices().take_while(|(_, c)| predicate(*c)).last().map(|(i, _)| i)
}
/// Check if characters is in a special characters list or not
fn is_symbolic(char: char) -> bool {
!char.is_whitespace() && !SPECIAL_CHARS.iter().any(|t| t.chars().any(|other| other == char))
}
/// Return type: (token, (start, end))
pub fn tokenize(src: &str) -> impl Iterator<Item = (&str, (usize, usize))> {
let mut skip: Option<usize> = None;
src.char_indices().filter_map(move |(i, char)| {
if skip.map(|dest| dest > i).unwrap_or(false) { return None; }
else { skip = None; }
// Whitespaces
if char.is_whitespace() { return None; }
// Special characters
for special in &SPECIAL_CHARS {
if match_front(&src[i..], special) {
skip = Some(i + special.len());
return Some((*special, (i, i + special.len())));
}
}
// Strings
if char == '"' {
let match_end = match_predicate(&src[i + 1..], |c| c != '"');
if let Some(end) = match_end {
let string_end = i + end + 3;
skip = Some(string_end);
return Some((&src[i..string_end], (i, string_end)));
}
}
// Comments
// Check if the current char is a semicolon and
if char == ';' && src[i + 1..].chars().next().map_or(false, |c| c == ';') {
// Get length until end of line
let end = i + 2 + match_predicate(&src[i + 2..], |c| c!= '\n').unwrap_or(0);
skip = Some(end + 1);
return None;
}
// Numbers
if char.is_numeric() {
let front = i + match_predicate(&src[i..], |c| c.is_numeric()).unwrap() + 1;
// Check if its a float (by checking if its contain a dot)
if front < src.len() - 1 && &src[front..front + 1] == "." {
let back = front + match_predicate(&src[front + 1..], |c| c.is_numeric()).unwrap() + 2;
skip = Some(back);
return Some((&src[i..back], (i, back)));
} else {
skip = Some(front);
return Some((&src[i..front], (i, front)));
}
}
// Symbols
if !char.is_numeric() && is_symbolic(char) {
let end = match_predicate(&src[i..], is_symbolic);
if let Some(last) = end {
let symbol_end = i + last + 1;
skip = Some(symbol_end);
return Some((&src[i..symbol_end], (i, symbol_end)));
}
}
None
})
}
// --- End tokenizer & Start parser ---
#[derive(Debug)]
pub enum ParseErrorKind {
UnexpectedParenClose,
}
impl fmt::Display for ParseErrorKind {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
ParseErrorKind::UnexpectedParenClose => write!(f, "Unexpected ')'"),
}
}
}
#[derive(Debug)]
pub struct ParseError {
pub kind: ParseErrorKind,
pub pos: (usize, usize),
}
impl ParseError {
fn new(kind: ParseErrorKind, pos: (usize, usize)) -> Self {
ParseError { kind, pos }
}
pub fn at(&self, src: &str) -> String {
let snip = &src[(self.pos.0.saturating_sub(5))..(if self.pos.0 + 5 > src.len() { src.len() } else { self.pos.0 + 5 })];
format!("\n{}..{}\n{}\nError: {} at {}", " ".repeat(3), snip, format!("{}^", " ".repeat(10)), self.kind, self.pos.0)
// Example:
//
// .."))) ) (pr
// ^
// Error: Unexpected ')' at 67
}
}
fn read<'a>(
tokens: impl Iterator<Item = (&'a str, (usize, usize))> + 'a
) -> impl Iterator<Item = Result<(Value, (usize, usize)), ParseError>> + 'a {
let mut stack: Vec<Tree> = Vec::new();
let mut parenths = 0;
let mut quote_next = false;
let mut block_start = 0;
tokens.filter_map(move |(token, (start, end))| {
match token {
"(" => {
parenths += 1;
if parenths == 1 {
block_start = start;
}
stack.push(Tree::List {
vec: Vec::new(),
quote: quote_next,
});
quote_next = false;
None
},
")" => {
parenths -= 1;
if stack.is_empty() {
Some(Err(ParseError::new(
ParseErrorKind::UnexpectedParenClose,
(start, end)
)))
} else {
let mut finished = stack.pop().unwrap();
if parenths == 0 {
stack = Vec::new();
let r = Some(Ok((finished.into_expr(), (block_start, end))));
block_start = 0;
r
} else {
let destination = stack.last_mut().unwrap();
if let Tree::List { vec, quote } = &finished {
if vec.is_empty() {
finished = Tree::Atom {
atom: Value::Nil,
quote: *quote,
};
}
}
if let Tree::List { vec, quote: _ } = destination { vec.push(finished); }
None
}
}
},
"'" => { quote_next = true; None },
_ => {
let expr = Tree::Atom {
atom: read_atom(token),
quote: quote_next,
};
quote_next = false;
if let Some(last) = stack.last_mut() {
if let Tree::List { vec, quote: _ } = last {
vec.push(expr);
}
None
} else {
Some(Ok((expr.into_expr(), (start, end))))
}
}
}
})
}
fn read_atom(token: &str) -> Value {
let lower = token.to_lowercase();
match lower.as_str() {
"true" => Value::True,
"false" => Value::False,
"nil" => Value::Nil,
_ => {
// Parse number
if let Ok(int) = token.parse::<i64>() { Value::Int(int) }
// Parse float
else if let Ok(float) = token.parse::<f64>() { Value::Float(float) }
// Parse string
else if token.chars().next().map_or(false, |c| c == '"') && token.chars().nth_back(0).map_or(false, |c| c == '"') {
Value::String(String::from(&token[1..token.chars().count() - 1]))
} else {
Value::Symbol(String::from(token))
}
}
}
}
// --- End parser ---
pub fn parse(src: &str) -> impl Iterator<Item = Result<(Value, (usize, usize)), ParseError>> + '_ {
read(tokenize(src))
}

View file

@ -1,44 +1,21 @@
use std::{fs::{ read_to_string, File }, io::Write}; use std::fs;
use clap::Parser;
use clap::Parser as ArgParser;
/// Arguments handler. /// Arguments handler.
pub mod args; pub mod args;
use args::{ Args, Options }; use args::{Args, Options};
/// A front-end for the compiler.
/// Contains parser and tokenizer.
/// TODO: Semantic analysis and Type checking.
pub mod front; pub mod front;
use front::parser::parse; use front::lex::Lexer;
/// A middle-end for the compiler.
/// Contains high intermediate representation (HIR).
pub mod middle;
use crate::middle::hir::to_hirs;
fn main() { fn main() {
let args = Args::parse(); let args = Args::parse();
match args.options { match args.options {
Options::Compile { input, ast } => { Options::Compile { input: src, ast: _print_ast } => {
let code = read_to_string(&input).unwrap(); let bytes: Vec<u8> = fs::read(src).unwrap();
let tree = parse(&code); let tokens = Lexer::lex_tokens(&bytes);
match ast { println!("{:?}", tokens);
true => for node in tree { println!("{:#?}", node) },
false => {
// Check if the tree is valid
let mut checked_tree = Vec::new();
for node in tree {
match node {
Ok(node) => checked_tree.push(node.0),
Err(err) => println!("{:?}", err),
}
};
// Convert the tree to HIR
let hir = to_hirs(&checked_tree);
println!("{:#?}", hir);
},
}
}, },
} }
} }

View file

@ -1,116 +0,0 @@
use std::{rc::Rc, borrow::Borrow};
use crate::front::parser::Value;
#[derive(Debug, Clone)]
pub enum HIRLiteral {
True, False, Nil,
Int(i64), Float(f64),
String(String), Symbol(String),
List(Box<HIRLiteral>, Vec<HIRLiteral>),
}
#[derive(Debug, Clone)]
pub enum HIR {
Declare { name: String, value: HIRLiteral },
Set { name: String, value: HIRLiteral },
Let { bindings: Vec<(String, HIR)>, body: Vec<HIR> },
If { condition: Box<HIR>, then: Box<HIR>, else_: Option<Box<HIR>> },
Call { func: String, args: Rc<Vec<HIR>> },
Quoted { body: HIRLiteral },
Literal(HIRLiteral),
}
pub fn to_hirs(ast: &Vec<Value>) -> Vec<HIR> {
let mut hir = Vec::new();
for node in ast {
match node {
Value::List(car, cdr) => {
match &*car.borrow() {
Value::Symbol(ref function) => {
match function.as_str() {
"quote" => {
hir.push(HIR::Quoted { body: to_hir_literal(&cdr[0].clone()) });
},
"if" => {
let cond = to_hir_single(&cdr[0].clone());
let then = to_hir_single(&cdr[1].clone());
let else_ = if cdr.len() > 2 { Some(Box::new(to_hir_single(&cdr[2].clone()))) }
else { None };
hir.push(HIR::If { condition: Box::new(cond), then: Box::new(then), else_ });
}
"def" => {
let name: String = match &cdr[0].clone() {
Value::Symbol(name) => name.clone(),
_ => panic!("def expects a symbol as first argument"),
};
let value = &cdr[1].clone();
hir.push(HIR::Declare { name, value: to_hir_literal(value) });
},
"print" => {
let value = &cdr[0].clone();
hir.push(HIR::Call { func: "print".to_string(), args: Rc::new(vec![to_hir_single(value)]) });
},
"equal" => {
let left = &cdr[0].clone();
let right = &cdr[1].clone();
hir.push(HIR::Call { func: "equal".to_string(), args: Rc::new(vec![to_hir_single(left), to_hir_single(right)]) });
},
_ => {
dbg!(function);
todo!();
}
} // --- End match `function` ---
},
_ => {
dbg!(car);
todo!();
} // --- End match `car` ---
}
},
_ => hir.push(to_hir_single(node)),
} // --- End match `node` ---
}
hir
}
fn to_hir_single(value: &Value) -> HIR {
match value {
Value::List(car, cdr) => {
let mut vec: Vec<Value> = Vec::new();
let list: Value = Value::List(car.clone(), cdr.clone());
vec.push(list);
let result = to_hirs(&vec)[0].clone();
result
},
_ => HIR::Literal(to_hir_literal(value)),
}
}
fn to_hir_literal(value: &Value) -> HIRLiteral {
match value {
Value::True => HIRLiteral::True,
Value::False => HIRLiteral::False,
Value::Int(i) => HIRLiteral::Int(*i),
Value::Float(fl) => HIRLiteral::Float(*fl),
Value::String(s) => HIRLiteral::String(s.clone()),
Value::Symbol(s) => HIRLiteral::Symbol(s.clone()),
Value::List(car, cdr) => {
let car_literal = to_hir_literal(&car);
let cdr_literal = cdr.iter().map(|a| to_hir_literal(a)).collect::<Vec<HIRLiteral>>();
HIRLiteral::List(Box::new(car_literal), cdr_literal)
},
Value::Nil => HIRLiteral::Nil,
}
}

View file

@ -1 +0,0 @@
pub mod hir;