From 3a26a02661d9b497538b77753cebcf5d2d10a77e Mon Sep 17 00:00:00 2001
From: Erin <erin@erindesu.cz>
Date: Tue, 3 Oct 2023 01:38:44 +0200
Subject: [PATCH] Lexer texts

---
 src/syntax/token.rs | 46 ++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 43 insertions(+), 3 deletions(-)

diff --git a/src/syntax/token.rs b/src/syntax/token.rs
index bbd8ad3..a9e6a8e 100644
--- a/src/syntax/token.rs
+++ b/src/syntax/token.rs
@@ -1,5 +1,5 @@
 use lasso::{Rodeo, Spur};
-use logos::Logos;
+use logos::{Lexer, Logos};
 
 #[derive(Default)]
 pub struct Extras {
@@ -50,14 +50,14 @@ pub enum Token {
 
     #[regex(
         r"\p{XID_Start}\p{XID_Continue}*",
-        |l| l.extras.interner.get_or_intern(l.slice()),
+        |l| intern(l, l.slice()),
     )] Ident(Spur),
 
     #[regex(
         "\"[^\"]*\"",
         |l| {
             let s = l.slice();
-            l.extras.interner.get_or_intern(&s[1..s.len() - 1])
+            intern(l, &s[1..s.len() - 1])
         },
     )] String(Spur),
 
@@ -66,3 +66,43 @@ pub enum Token {
         |l| l.slice().parse::<u64>().ok()
     )] Int(u64),
 }
+
+fn intern(lexer: &mut Lexer<'_, Token>, s: &str) -> Spur {
+    lexer.extras.interner.get_or_intern(s)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn ident() {
+        let mut lexer = Token::lexer("いえぶる able");
+        assert_eq!(
+            lexer.next(),
+            Some(Ok(Token::Ident(intern(&mut lexer, "いえぶる")))),
+        );
+
+        assert_eq!(
+            lexer.next(),
+            Some(Ok(Token::Ident(intern(&mut lexer, "able")))),
+        );
+    }
+
+    #[test]
+    fn string() {
+        let mut lexer = Token::lexer("\"sussy\" \"baka");
+        assert_eq!(
+            lexer.next(),
+            Some(Ok(Token::String(intern(&mut lexer, "sussy")))),
+        );
+
+        assert_eq!(lexer.next(), Some(Err(())),);
+    }
+
+    #[test]
+    fn symbol_alt() {
+        let mut lexer = Token::lexer("-> →");
+        assert_eq!(lexer.next(), lexer.next());
+    }
+}