Lexer texts

src/syntax/token.rs

@@ -1,5 +1,5 @@
 use lasso::{Rodeo, Spur};
-use logos::Logos;
+use logos::{Lexer, Logos};
 
 #[derive(Default)]
 pub struct Extras {
@@ -50,14 +50,14 @@ pub enum Token {
 
     #[regex(
         r"\p{XID_Start}\p{XID_Continue}*",
-        |l| l.extras.interner.get_or_intern(l.slice()),
+        |l| intern(l, l.slice()),
     )] Ident(Spur),
 
     #[regex(
         "\"[^\"]*\"",
         |l| {
             let s = l.slice();
-            l.extras.interner.get_or_intern(&s[1..s.len() - 1])
+            intern(l, &s[1..s.len() - 1])
         },
     )] String(Spur),
 
@@ -66,3 +66,43 @@ pub enum Token {
         |l| l.slice().parse::<u64>().ok()
     )] Int(u64),
 }
+
+fn intern(lexer: &mut Lexer<'_, Token>, s: &str) -> Spur {
+    lexer.extras.interner.get_or_intern(s)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn ident() {
+        let mut lexer = Token::lexer("いえぶる able");
+        assert_eq!(
+            lexer.next(),
+            Some(Ok(Token::Ident(intern(&mut lexer, "いえぶる")))),
+        );
+
+        assert_eq!(
+            lexer.next(),
+            Some(Ok(Token::Ident(intern(&mut lexer, "able")))),
+        );
+    }
+
+    #[test]
+    fn string() {
+        let mut lexer = Token::lexer("\"sussy\" \"baka");
+        assert_eq!(
+            lexer.next(),
+            Some(Ok(Token::String(intern(&mut lexer, "sussy")))),
+        );
+
+        assert_eq!(lexer.next(), Some(Err(())),);
+    }
+
+    #[test]
+    fn symbol_alt() {
+        let mut lexer = Token::lexer("-> →");
+        assert_eq!(lexer.next(), lexer.next());
+    }
+}