use logos::{Lexer, Logos}; #[derive(Logos, Debug, PartialEq, Clone)] pub enum Token { // Symbols #[token("(")] LeftParen, #[token(")")] RightParen, #[token("[")] LeftBracket, #[token("]")] RightBracket, #[token("{")] LeftCurly, #[token("}")] RightCurly, #[token(";")] Semicolon, #[token(",")] Comma, // Operators #[token("+")] Plus, #[token("-")] Minus, #[token("*")] Star, #[token("/")] FwdSlash, #[token("=:")] Assign, #[token("<=")] Arrow, // Logical operators #[token("<")] LessThan, #[token(">")] GreaterThan, #[token("=")] Equals, #[token("ain't")] Aint, // Keywords #[token("functio")] Functio, /// Brain fuck FFI #[token("bff")] Bff, /// Variable bro #[token("dim")] Dim, /// Prints the preceding things #[token("print")] Print, /// Read input into preceding variable #[token("read")] Read, /// Ban the following variable from ever being used again #[token("melo")] Melo, #[token("T-Dark")] TDark, // Control flow keywords #[token("unless")] Unless, #[token("loop")] Loop, #[token("break")] Break, /// HopBack hops on the back of loop - like `continue` #[token("hopback")] HopBack, /// Crash with random error (see discussion #17) #[token("rlyeh")] Rlyeh, #[token("rickroll")] Rickroll, // Literals /// String #[token("/*", get_string)] String(String), /// Integer #[regex(r"-?[0-9]+", get_value)] Integer(isize), // A character (to be base-55 converted) #[regex(r"\p{XID_Start}", get_value)] Char(char), /// An identifier #[regex(r"\p{XID_Start}[\p{XID_Continue}]+", get_ident)] Identifier(String), #[regex(r"owo .*")] Comment, #[regex(r"[ \t\n\f]+", logos::skip)] #[error] Error, } fn get_value(lexer: &mut Lexer) -> Option { lexer.slice().parse().ok() } fn get_string(lexer: &mut Lexer) -> Option { lexer.bump(lexer.remainder().find("*/")?); let string = lexer.slice()[2..].to_owned(); lexer.bump(2); Some(string) } fn get_ident(lexer: &mut Lexer) -> String { lexer.slice().to_owned() } #[cfg(test)] mod tests { use super::Token; use super::Token::*; use logos::Logos; #[test] fn simple_fn() { let code = "functio test() { dim var 3; unless (var ain't 3) { var print } }"; let expected = &[ Functio, Identifier("test".to_owned()), LeftParen, RightParen, LeftCurly, Dim, Identifier("var".to_owned()), Integer(3), Semicolon, Unless, LeftParen, Identifier("var".to_owned()), Aint, Integer(3), RightParen, LeftCurly, Identifier("var".to_owned()), Print, RightCurly, RightCurly, ]; let lexer = Token::lexer(code); let result: Vec = lexer.collect(); assert_eq!(result, expected); } }