Added lexer

This commit is contained in:
Erin 2023-10-03 01:21:47 +02:00 committed by ondra05
parent a9a1e22760
commit 399bd4f6a1
3 changed files with 70 additions and 0 deletions

View file

@ -1,5 +1,6 @@
// Rhea // Rhea
mod syntax;
mod utils; mod utils;
use std::io::{stdin, Read}; use std::io::{stdin, Read};

1
src/syntax/mod.rs Normal file
View file

@ -0,0 +1 @@
pub mod token;

68
src/syntax/token.rs Normal file
View file

@ -0,0 +1,68 @@
use lasso::{Rodeo, Spur};
use logos::Logos;
#[derive(Default)]
pub struct Extras {
pub interner: Rodeo,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Logos)]
#[logos(extras = Extras)]
#[logos(skip r"[ \t\n\f]+")]
#[logos(skip r"\\.*")]
#[rustfmt::skip]
pub enum Token {
#[token("(")] LeftParen,
#[token(")")] RightParen,
#[token("{")] LeftCurly,
#[token("}")] RightCurly,
#[token(".")] Dot,
#[token(",")] Comma,
#[token(":")] Colon,
#[token(";")] Semicolon,
#[token("")] //______
#[token("<-")] LArrow,
#[token("")] //______
#[token("->")] RArrow,
#[token("+")] Plus,
#[token("-")] Minus,
#[token("*")] Star,
#[token("/")] Slash,
#[token("=")] Equ,
#[token("")] //___
#[token("/=")] Neq,
#[token("<")] Lt,
#[token(">")] Gt,
#[token("")] //____
#[token("<=")] LtEq,
#[token("")] //____
#[token(">=")] GtEq,
#[token("func")] Func,
#[token("var")] Var,
#[token("const")] Const,
#[token("include")] Include,
#[token("switch")] Switch,
#[token("loop")] Loop,
#[regex(
r"\p{XID_Start}\p{XID_Continue}*",
|l| l.extras.interner.get_or_intern(l.slice()),
)] Ident(Spur),
#[regex(
"\"[^\"]*\"",
|l| {
let s = l.slice();
l.extras.interner.get_or_intern(&s[1..s.len() - 1])
},
)] String(Spur),
#[regex(
"[0-9]+",
|l| l.slice().parse::<u64>().ok()
)] Int(u64),
}