Compare commits
No commits in common. "rewrite" and "master" have entirely different histories.
28
Cargo.lock
generated
28
Cargo.lock
generated
|
@ -80,26 +80,6 @@ version = "0.2.148"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b"
|
||||
|
||||
[[package]]
|
||||
name = "literify"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fd552332051e9b3db140d34a371dcc0ed378b72a9227b5273070af58ea34abf4"
|
||||
dependencies = [
|
||||
"litrs",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litrs"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4f17c3668f3cc1132437cdadc93dab05e52d592f06948d3f64828430c36e4a70"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "logos"
|
||||
version = "0.13.0"
|
||||
|
@ -138,12 +118,6 @@ version = "1.18.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
|
||||
|
||||
[[package]]
|
||||
name = "paste"
|
||||
version = "1.0.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.67"
|
||||
|
@ -184,9 +158,7 @@ dependencies = [
|
|||
"bumpalo",
|
||||
"chumsky",
|
||||
"lasso",
|
||||
"literify",
|
||||
"logos",
|
||||
"paste",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
|
@ -7,6 +7,4 @@ edition = "2021"
|
|||
bumpalo = { version = "3", features = ["collections"] }
|
||||
chumsky = "1.0.0-alpha"
|
||||
lasso = "0.7"
|
||||
literify = "0.2"
|
||||
logos = "0.13"
|
||||
paste = "1.0"
|
||||
|
|
|
@ -1,3 +0,0 @@
|
|||
enum_discrim_align_threshold = 16
|
||||
struct_field_align_threshold = 16
|
||||
imports_granularity = "one"
|
13
src/main.rs
13
src/main.rs
|
@ -1,22 +1,21 @@
|
|||
// Rhea
|
||||
|
||||
use {logos::Logos, syntax::token::Token};
|
||||
use bumpalo::Bump;
|
||||
use logos::Logos;
|
||||
use std::io::{stdin, Read};
|
||||
use utils::default;
|
||||
|
||||
mod syntax;
|
||||
mod utils;
|
||||
|
||||
use {
|
||||
bumpalo::Bump,
|
||||
std::io::{stdin, Read},
|
||||
utils::default,
|
||||
};
|
||||
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let mut buf = default();
|
||||
stdin().read_to_string(&mut buf)?;
|
||||
|
||||
let lexer = syntax::token::Token::lexer_with_extras(&buf, default());
|
||||
let arena = Bump::new();
|
||||
syntax::parser::parse_lexer(Token::lexer(&buf), &arena);
|
||||
syntax::parser::parse_lexer(lexer, &arena);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
use {chumsky::span::SimpleSpan, lasso::Spur};
|
||||
use {super::token::IntLit, chumsky::span::SimpleSpan, lasso::Spur};
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct Spanned<T> {
|
||||
|
@ -8,16 +8,9 @@ pub struct Spanned<T> {
|
|||
|
||||
impl<T> Spanned<T> {
|
||||
#[inline]
|
||||
pub const fn new(item: T, span: SimpleSpan) -> Self {
|
||||
pub fn new(item: T, span: SimpleSpan) -> Self {
|
||||
Self { item, span }
|
||||
}
|
||||
|
||||
pub fn map<U>(self, mut f: impl FnMut(T) -> U) -> Spanned<U> {
|
||||
Spanned {
|
||||
item: f(self.item),
|
||||
span: self.span,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub type SpanExpr<'a> = Spanned<Expr<'a>>;
|
||||
|
@ -31,82 +24,60 @@ pub enum Expr<'a> {
|
|||
Path(ExprList<'a>),
|
||||
Literal(Literal),
|
||||
Call(ExprRef<'a>, ExprList<'a>),
|
||||
Binary(Spanned<BinaryOp>, ExprRef<'a>, ExprRef<'a>),
|
||||
Unary(Spanned<UnaryOp>, ExprRef<'a>),
|
||||
Binary(Spanned<BinaryOperator>, ExprRef<'a>, ExprRef<'a>),
|
||||
Unary(Spanned<UnaryOperator>, ExprRef<'a>),
|
||||
BindLocal(Spanned<Pattern>, ExprRef<'a>, Option<ExprList<'a>>),
|
||||
BindIn(
|
||||
Spanned<Pattern>,
|
||||
ExprRef<'a>,
|
||||
ExprList<'a>,
|
||||
Option<ExprList<'a>>,
|
||||
),
|
||||
Set(ExprRef<'a>, ExprRef<'a>),
|
||||
Loop(ExprList<'a>),
|
||||
Match(ExprRef<'a>, &'a [(Spanned<Pattern>, SpanExpr<'a>)]),
|
||||
Func(&'a [(Spanned<Pattern>, Spanned<Type>)], Spanned<Type>, ExprRef<'a>),
|
||||
Block(ExprList<'a>),
|
||||
Definition(Definition<'a>),
|
||||
Switch {
|
||||
on: ExprRef<'a>,
|
||||
branches: &'a [(Spanned<Switcher>, ExprRef<'a>)],
|
||||
else_: ExprRef<'a>,
|
||||
},
|
||||
CondSwitch {
|
||||
branches: &'a [(ExprRef<'a>, ExprRef<'a>)],
|
||||
else_: ExprRef<'a>,
|
||||
},
|
||||
Break(Option<ExprRef<'a>>),
|
||||
Return(Option<ExprRef<'a>>),
|
||||
Uninit,
|
||||
Continue,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Definition<'a> {
|
||||
Binding {
|
||||
kind: DefKind,
|
||||
ident: Spanned<Ident>,
|
||||
ty: Option<Spanned<Type>>,
|
||||
init: ExprRef<'a>,
|
||||
},
|
||||
Func {
|
||||
name: Spanned<Ident>,
|
||||
params: &'a [(Spanned<Ident>, Spanned<Type>)],
|
||||
ret: Spanned<Type>,
|
||||
body: ExprList<'a>,
|
||||
},
|
||||
Unit,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Type {
|
||||
Ident(Ident),
|
||||
Unit,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Switcher {
|
||||
pub enum Pattern {
|
||||
Ident(Ident),
|
||||
Literal(Literal),
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum DefKind {
|
||||
Const,
|
||||
Var,
|
||||
None,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Literal {
|
||||
String(Spur),
|
||||
Int(u64),
|
||||
Integer(IntLit),
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum BinaryOp {
|
||||
pub enum BinaryOperator {
|
||||
Plus,
|
||||
Minus,
|
||||
Star,
|
||||
Slash,
|
||||
Equ,
|
||||
Neq,
|
||||
And,
|
||||
VLine,
|
||||
Lt,
|
||||
Gt,
|
||||
LtEq,
|
||||
GtEq,
|
||||
Equ,
|
||||
Nequ,
|
||||
LtEqu,
|
||||
GtEqu,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum UnaryOp {
|
||||
pub enum UnaryOperator {
|
||||
Tilde,
|
||||
Minus,
|
||||
Not,
|
||||
Star,
|
||||
}
|
||||
|
|
|
@ -1,9 +1,9 @@
|
|||
use super::ast::{DefKind, Expr, ExprList, Type};
|
||||
use super::ast::Type;
|
||||
|
||||
use {
|
||||
super::{
|
||||
ast::{Definition, Ident, SpanExpr, Spanned},
|
||||
token::{Token, T},
|
||||
ast::{BinaryOperator, Expr, Literal, Pattern, SpanExpr, Spanned, UnaryOperator},
|
||||
token::Token,
|
||||
},
|
||||
crate::utils::Pipe,
|
||||
bumpalo::Bump,
|
||||
|
@ -23,70 +23,195 @@ macro_rules! equivmap {
|
|||
};
|
||||
}
|
||||
|
||||
fn ident<'a, I>() -> impl Parser<'a, I, Spanned<Ident>, Extra<'a>> + Clone + Copy
|
||||
fn expr<'a, I>() -> impl Parser<'a, I, SpanExpr<'a>, Extra<'a>> + Clone
|
||||
where
|
||||
I: Input<'a, Token = Token, Span = SimpleSpan> + ValueInput<'a>,
|
||||
{
|
||||
select!(Token::Ident(id) => id).map_with_span(Spanned::new)
|
||||
recursive(|expr| {
|
||||
let ident = select!(Token::Ident(id) => id);
|
||||
|
||||
let literal = select! {
|
||||
Token::Int(a) => Literal::Integer(a),
|
||||
Token::String(a) => Literal::String(a)
|
||||
};
|
||||
|
||||
let pattern = select! {
|
||||
Token::Ident(id) => Pattern::Ident(id),
|
||||
Token::Underscore => Pattern::None,
|
||||
}
|
||||
.or(literal.map(Pattern::Literal))
|
||||
.map_with_span(Spanned::new);
|
||||
|
||||
fn ty<'a, I>() -> impl Parser<'a, I, Spanned<Type>, Extra<'a>> + Clone + Copy
|
||||
where
|
||||
I: Input<'a, Token = Token, Span = SimpleSpan> + ValueInput<'a>,
|
||||
{
|
||||
ident().map(|i| i.map(Type::Ident))
|
||||
}
|
||||
let type_ = just([Token::LeftParen, Token::RightParen])
|
||||
.to(Type::Unit)
|
||||
.or(ident.map(Type::Ident))
|
||||
.map_with_span(Spanned::new);
|
||||
|
||||
fn definition<'a, I>() -> impl Parser<'a, I, Spanned<Definition<'a>>, Extra<'a>> + Clone
|
||||
where
|
||||
I: Input<'a, Token = Token, Span = SimpleSpan> + ValueInput<'a>,
|
||||
{
|
||||
let ident = ident();
|
||||
let ty = ty();
|
||||
|
||||
let func = just(T!["func"])
|
||||
.ignore_then(ident)
|
||||
.then(
|
||||
ident
|
||||
.then_ignore(just(T![":"]))
|
||||
.then(ty)
|
||||
.separated_by(just(T![","]))
|
||||
let block = expr
|
||||
.clone()
|
||||
.separated_by(just(Token::Semicolon))
|
||||
.allow_trailing()
|
||||
.pipe(arena_collect)
|
||||
.delimited_by(just(T!["("]), just(T![")"])),
|
||||
)
|
||||
.then_ignore(just(T!["→"]))
|
||||
.then(ty)
|
||||
.then(just([T!["{"], T!["}"]]))
|
||||
.map_with_state(
|
||||
|(((name, params), ret), _body), _, state| Definition::Func {
|
||||
name,
|
||||
params,
|
||||
ret,
|
||||
body: state.arena.alloc_slice_copy(&[]),
|
||||
},
|
||||
);
|
||||
.delimited_by(just(Token::LeftCurly), just(Token::RightCurly));
|
||||
|
||||
let binding = equivmap!(Token, DefKind, [Const, Var])
|
||||
.then(ident)
|
||||
.then(just(T![":"]).ignore_then(ty).or_not())
|
||||
let func = just(Token::Func)
|
||||
.ignore_then(
|
||||
pattern
|
||||
.then_ignore(just(Token::Colon))
|
||||
.then(type_)
|
||||
.separated_by(just(Token::Comma))
|
||||
.allow_trailing()
|
||||
.pipe(arena_collect)
|
||||
.delimited_by(just(Token::LeftParen), just(Token::RightParen)),
|
||||
)
|
||||
.then_ignore(just(Token::Colon))
|
||||
.then(type_)
|
||||
.then(
|
||||
just(T!["="]).ignore_then(
|
||||
just(T!["uninit"])
|
||||
.to(Expr::Uninit)
|
||||
.map_with_span(Spanned::new),
|
||||
),
|
||||
just(Token::Equ)
|
||||
.ignore_then(expr.clone())
|
||||
.or(block.clone().map(Expr::Block).map_with_span(Spanned::new)),
|
||||
)
|
||||
.map_with_state(|((params, ret), expr), _, state| {
|
||||
Expr::Func(params, ret, state.arena.alloc(expr))
|
||||
});
|
||||
|
||||
let atom = literal
|
||||
.map(Expr::Literal)
|
||||
.or(just([Token::LeftParen, Token::RightParen]).to(Expr::Unit))
|
||||
.or(ident.map(Expr::Ident))
|
||||
.or(func)
|
||||
.map_with_span(Spanned::new)
|
||||
.or(expr
|
||||
.clone()
|
||||
.delimited_by(just(Token::LeftParen), just(Token::RightParen)));
|
||||
|
||||
// <expr>(expr1, expr2, …)
|
||||
let call = atom.clone().foldl_with_state(
|
||||
expr.clone()
|
||||
.separated_by(just(Token::Comma))
|
||||
.allow_trailing()
|
||||
.pipe(arena_collect)
|
||||
.delimited_by(just(Token::LeftParen), just(Token::RightParen))
|
||||
.map_with_span(Spanned::new)
|
||||
.repeated(),
|
||||
|expr, paramlist, state: &mut State| {
|
||||
Spanned::new(
|
||||
Expr::Call(state.arena.alloc(expr), paramlist.item),
|
||||
merge_spans(expr.span, paramlist.span),
|
||||
)
|
||||
.map_with_state(
|
||||
|(((kind, ident), ty), init), _, state| Definition::Binding {
|
||||
kind,
|
||||
ident,
|
||||
ty,
|
||||
init: state.arena.alloc(init),
|
||||
},
|
||||
);
|
||||
|
||||
func.or(binding).map_with_span(Spanned::new)
|
||||
let path = call
|
||||
.clone()
|
||||
.map_with_state(|item, _, state| bumpalo::vec![in state.arena; item])
|
||||
.foldl(
|
||||
just(Token::Dot).ignore_then(call).repeated(),
|
||||
|mut v, expr| {
|
||||
v.push(expr);
|
||||
v
|
||||
},
|
||||
)
|
||||
.map(|v| Expr::Path(v.into_bump_slice()))
|
||||
.map_with_span(Spanned::new);
|
||||
|
||||
/* let unary = equivmap!(Token, UnaryOperator, [Minus, Tilde])
|
||||
.map_with_span(Spanned::new)
|
||||
.repeated()
|
||||
.foldr_with_state(call, |op, expr, state| {
|
||||
Spanned::new(
|
||||
Expr::Unary(op, state.arena.alloc(expr)),
|
||||
merge_spans(op.span, expr.span),
|
||||
)
|
||||
});
|
||||
*/
|
||||
|
||||
let unary = path.foldl_with_state(
|
||||
just([Token::Dot, Token::Star])
|
||||
.to(UnaryOperator::Star)
|
||||
.or(just(Token::Tilde).to(UnaryOperator::Tilde))
|
||||
.map_with_span(Spanned::new)
|
||||
.repeated(),
|
||||
|expr, op, state| {
|
||||
Spanned::new(
|
||||
Expr::Unary(op, state.arena.alloc(expr)),
|
||||
merge_spans(expr.span, op.span),
|
||||
)
|
||||
},
|
||||
);
|
||||
|
||||
// <exprL> OP <exprR>
|
||||
let binary = unary.clone().foldl_with_state(
|
||||
equivmap!(
|
||||
Token,
|
||||
BinaryOperator,
|
||||
[Plus, Minus, Star, Slash, And, VLine, Lt, Gt, Equ, Nequ, LtEqu, GtEqu],
|
||||
)
|
||||
.map_with_span(Spanned::new)
|
||||
.then(unary)
|
||||
.repeated(),
|
||||
|l, (op, r), state: &mut State| {
|
||||
Spanned::new(
|
||||
Expr::Binary(op, state.arena.alloc(l), state.arena.alloc(r)),
|
||||
merge_spans(l.span, r.span),
|
||||
)
|
||||
},
|
||||
);
|
||||
|
||||
let bind = {
|
||||
let start = pattern.then_ignore(just(Token::Colon)).then(expr.clone()); // <pat> := <expr>
|
||||
let else_ = just(Token::Else).ignore_then(block.clone()).or_not(); // else {…}
|
||||
|
||||
// <pat> := <expr> [else {…}]
|
||||
let local = start.clone().then(else_.clone()).map_with_state(
|
||||
|((pat, expr), else_), _, state| {
|
||||
Expr::BindLocal(pat, &*state.arena.alloc(expr), else_)
|
||||
},
|
||||
);
|
||||
|
||||
// <pat> := <expr> {…} else {…}
|
||||
let in_ = start.then(block.clone()).then(else_).map_with_state(
|
||||
|(((pat, expr), block), else_), _, state| {
|
||||
Expr::BindIn(pat, &*state.arena.alloc(expr), block, else_)
|
||||
},
|
||||
);
|
||||
|
||||
in_.or(local)
|
||||
};
|
||||
|
||||
// <atom> ← <expr>
|
||||
let set = atom
|
||||
.clone()
|
||||
.then_ignore(just(Token::LArrow))
|
||||
.then(expr.clone())
|
||||
.map_with_state(|(place, expr), _, state| {
|
||||
Expr::Set(state.arena.alloc(place), state.arena.alloc(expr))
|
||||
});
|
||||
|
||||
// <expr>.match { <pat> → <expr>, … }
|
||||
let match_ = atom
|
||||
.clone()
|
||||
.then_ignore(just([Token::Dot, Token::Match]))
|
||||
.then(
|
||||
pattern
|
||||
.then_ignore(just(Token::RArrow))
|
||||
.then(expr)
|
||||
.separated_by(just(Token::Comma))
|
||||
.allow_trailing()
|
||||
.pipe(arena_collect)
|
||||
.delimited_by(just(Token::LeftCurly), just(Token::RightCurly)),
|
||||
)
|
||||
.map_with_state(|(expr, branches), _, state| {
|
||||
Expr::Match(state.arena.alloc(expr), branches)
|
||||
});
|
||||
|
||||
bind.or(set)
|
||||
.or(match_)
|
||||
.or(block.map(Expr::Block))
|
||||
.map_with_span(Spanned::new)
|
||||
.or(binary)
|
||||
.or(atom)
|
||||
})
|
||||
}
|
||||
|
||||
pub struct State<'a> {
|
||||
|
@ -102,8 +227,8 @@ pub fn parse_input<'a>(
|
|||
) -> ParseResult {
|
||||
println!(
|
||||
"{:?}",
|
||||
definition()
|
||||
.separated_by(just(T![";"]))
|
||||
expr()
|
||||
.separated_by(just(Token::Semicolon))
|
||||
.allow_trailing()
|
||||
.pipe(arena_collect)
|
||||
.parse_with_state(input, &mut State { arena })
|
||||
|
|
|
@ -1,139 +1,112 @@
|
|||
use lasso::{Rodeo, Spur};
|
||||
use logos::{Lexer, Logos};
|
||||
use lasso::Spur;
|
||||
use logos::Lexer;
|
||||
|
||||
use {lasso::Rodeo, logos::Logos};
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct Extras {
|
||||
pub struct Lextras {
|
||||
pub interner: Rodeo,
|
||||
}
|
||||
|
||||
macro_rules! token_def {
|
||||
(
|
||||
unit { $($u_name:ident : $($u_tok:literal),* $(,)?;)* }
|
||||
keyword { $($kw:tt),* $(,)* }
|
||||
else { $($e_tt:tt)* }
|
||||
) => {
|
||||
literify::literify!(paste::paste! {
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Logos)]
|
||||
#[logos(extras = Extras)]
|
||||
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
|
||||
pub enum IntLit {
|
||||
Signed(i64),
|
||||
Unsigned(u64),
|
||||
}
|
||||
|
||||
#[derive(Logos, Copy, Clone, Debug, PartialEq, Eq)]
|
||||
#[logos(extras = Lextras)]
|
||||
#[logos(skip r"[ \t\n\f]+")]
|
||||
#[logos(skip r"\\.*")]
|
||||
#[logos(skip r"-- .*")]
|
||||
#[rustfmt::skip]
|
||||
pub enum Token {
|
||||
$(
|
||||
$(#[token($u_tok)])*
|
||||
$u_name,
|
||||
)*
|
||||
#[token("(")] LeftParen,
|
||||
#[token(")")] RightParen,
|
||||
#[token("{")] LeftCurly,
|
||||
#[token("}")] RightCurly,
|
||||
#[token(".")] Dot,
|
||||
#[token(",")] Comma,
|
||||
#[token(":")] Colon,
|
||||
#[token(";")] Semicolon,
|
||||
#[token("_")] Underscore,
|
||||
|
||||
$(
|
||||
#[token(~($kw))]
|
||||
[<$kw:camel>],
|
||||
)*
|
||||
#[token("←")] //____
|
||||
#[token("<-")] LArrow,
|
||||
#[token("→")] //____
|
||||
#[token("->")] RArrow,
|
||||
|
||||
$($e_tt)*
|
||||
}
|
||||
#[token(":>")] Pipe,
|
||||
|
||||
macro_rules! T {
|
||||
$($(
|
||||
($u_tok) => { $crate::syntax::token::Token::$u_name };
|
||||
)*)*
|
||||
#[token("+")] Plus,
|
||||
#[token("-")] Minus,
|
||||
#[token("*")] Star,
|
||||
#[token("/")] Slash,
|
||||
#[token("&")] And,
|
||||
#[token("|")] VLine,
|
||||
#[token("~")] Tilde,
|
||||
|
||||
$((~($kw)) => { $crate::syntax::token::Token::[<$kw:camel>] };)*
|
||||
}
|
||||
});
|
||||
};
|
||||
}
|
||||
#[token("<")] Lt,
|
||||
#[token(">")] Gt,
|
||||
#[token("=")] Equ,
|
||||
#[token("≠") ] //__
|
||||
#[token("/=")] Nequ,
|
||||
#[token("≤") ] //___
|
||||
#[token("<=")] LtEqu,
|
||||
#[token("≥") ] //___,
|
||||
#[token(">=")] GtEqu,
|
||||
|
||||
token_def!(
|
||||
unit {
|
||||
LeftParen : "(";
|
||||
RightParen: ")";
|
||||
LeftCurly : "{";
|
||||
RightCurly: "}";
|
||||
Dot : ".";
|
||||
Comma : ",";
|
||||
Colon : ":";
|
||||
Semicolon : ";";
|
||||
#[token("match")] Match,
|
||||
#[token("else")] Else,
|
||||
#[token("loop")] Loop,
|
||||
#[token("const")] Const,
|
||||
#[token("var")] Var,
|
||||
#[token("func")] Func,
|
||||
// Modules aren't real here ondra just variables with imported functions
|
||||
#[token("module")] Module,
|
||||
|
||||
LArrow: "←", "<-";
|
||||
RArrow: "→", "->";
|
||||
|
||||
Plus : "+";
|
||||
Minus : "-";
|
||||
Star : "*";
|
||||
Slash : "/";
|
||||
Precent: "%";
|
||||
|
||||
Equ : "=";
|
||||
Neq : "≠", "/=";
|
||||
Lt : "<";
|
||||
Gt : ">";
|
||||
LtEq : "≤", "<=";
|
||||
GtEq : "≥", ">=";
|
||||
}
|
||||
|
||||
keyword { func, var, const, include, switch, loop,
|
||||
return, break, continue, uninit, asm }
|
||||
|
||||
else {
|
||||
#[regex(
|
||||
r"\p{XID_Start}\p{XID_Continue}*",
|
||||
|l| intern(l, l.slice()),
|
||||
|l| l.extras.interner.get_or_intern(l.slice())
|
||||
)] Ident(Spur),
|
||||
|
||||
#[token("»", better_string)]
|
||||
#[regex(
|
||||
"\"[^\"]*\"",
|
||||
|l| {
|
||||
let s = l.slice();
|
||||
intern(l, &s[1..s.len() - 1])
|
||||
},
|
||||
let slice = l.slice();
|
||||
l.extras.interner.get_or_intern(&slice[1..slice.len() - 1])
|
||||
}
|
||||
)] String(Spur),
|
||||
|
||||
#[regex(
|
||||
"[0-9]+",
|
||||
|l| l.slice().parse::<u64>().ok()
|
||||
)] Int(u64),
|
||||
"-?[0-9]+",
|
||||
|l| {
|
||||
Some(if let Some(slice) = l.slice().strip_prefix('-') {
|
||||
IntLit::Signed(slice.parse::<i64>().ok()?)
|
||||
} else {
|
||||
IntLit::Unsigned(l.slice().parse::<u64>().ok()?)
|
||||
})
|
||||
}
|
||||
)] Int(IntLit),
|
||||
|
||||
Invalid,
|
||||
}
|
||||
);
|
||||
|
||||
pub(crate) use T;
|
||||
|
||||
fn intern(lexer: &mut Lexer<'_, Token>, s: &str) -> Spur {
|
||||
lexer.extras.interner.get_or_intern(s)
|
||||
// For Evy, with love.
|
||||
fn better_string(lexer: &mut Lexer<Token>) -> Option<Spur> {
|
||||
let mut count = 1;
|
||||
for (ix, chr) in lexer.remainder().char_indices() {
|
||||
match chr {
|
||||
'«' => count -= 1,
|
||||
'»' => count += 1,
|
||||
_ => (),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn ident() {
|
||||
let mut lexer = Token::lexer("いえぶる able");
|
||||
assert_eq!(
|
||||
lexer.next(),
|
||||
Some(Ok(Token::Ident(intern(&mut lexer, "いえぶる")))),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
lexer.next(),
|
||||
Some(Ok(Token::Ident(intern(&mut lexer, "able")))),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn string() {
|
||||
let mut lexer = Token::lexer("\"sussy\" \"baka");
|
||||
assert_eq!(
|
||||
lexer.next(),
|
||||
Some(Ok(Token::String(intern(&mut lexer, "sussy")))),
|
||||
);
|
||||
|
||||
assert_eq!(lexer.next(), Some(Err(())),);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn symbol_alt() {
|
||||
let mut lexer = Token::lexer("-> →");
|
||||
assert_eq!(lexer.next(), lexer.next());
|
||||
if count == 0 {
|
||||
let slice = &lexer.remainder()[..ix];
|
||||
lexer.bump(ix + '«'.len_utf8());
|
||||
return Some(lexer.extras.interner.get_or_intern(slice));
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue