Added lexer

src/main.rs

@@ -1,5 +1,6 @@
 // Rhea
 
+mod syntax;
 mod utils;
 
 use std::io::{stdin, Read};

src/syntax/mod.rs

@@ -0,0 +1 @@
+pub mod token;

src/syntax/token.rs

@@ -0,0 +1,68 @@
+use lasso::{Rodeo, Spur};
+use logos::Logos;
+
+#[derive(Default)]
+pub struct Extras {
+    pub interner: Rodeo,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Logos)]
+#[logos(extras = Extras)]
+#[logos(skip r"[ \t\n\f]+")]
+#[logos(skip r"\\.*")]
+#[rustfmt::skip]
+pub enum Token {
+    #[token("(")] LeftParen,
+    #[token(")")] RightParen,
+    #[token("{")] LeftCurly,
+    #[token("}")] RightCurly,
+    #[token(".")] Dot,
+    #[token(",")] Comma,
+    #[token(":")] Colon,
+    #[token(";")] Semicolon,
+
+    #[token("←")] //______
+    #[token("<-")] LArrow,
+    #[token("→")] //______
+    #[token("->")] RArrow,
+
+    #[token("+")] Plus,
+    #[token("-")] Minus,
+    #[token("*")] Star,
+    #[token("/")] Slash,
+
+    #[token("=")] Equ,
+    #[token("≠")] //___
+    #[token("/=")] Neq,
+    #[token("<")] Lt,
+    #[token(">")] Gt,
+    #[token("≤")] //____
+    #[token("<=")] LtEq,
+    #[token("≥")] //____
+    #[token(">=")] GtEq,
+
+    #[token("func")] Func,
+    #[token("var")] Var,
+    #[token("const")] Const,
+    #[token("include")] Include,
+    #[token("switch")] Switch,
+    #[token("loop")] Loop,
+
+    #[regex(
+        r"\p{XID_Start}\p{XID_Continue}*",
+        |l| l.extras.interner.get_or_intern(l.slice()),
+    )] Ident(Spur),
+
+    #[regex(
+        "\"[^\"]*\"",
+        |l| {
+            let s = l.slice();
+            l.extras.interner.get_or_intern(&s[1..s.len() - 1])
+        },
+    )] String(Spur),
+
+    #[regex(
+        "[0-9]+",
+        |l| l.slice().parse::<u64>().ok()
+    )] Int(u64),
+}