138 lines
3.2 KiB
Rust
138 lines
3.2 KiB
Rust
//! # elf_lojban
|
|
//!
|
|
//! elf_lojban: lex your lojban. Parser coming soon.
|
|
//!
|
|
//! ## Basic Usage
|
|
//!
|
|
//! Tokens can be lexed using the `lex` or `lex_stripped` functions:
|
|
//!
|
|
//! ```
|
|
//! use elf_lojban::{lex, lex_stripped};
|
|
//!
|
|
//! // Lex including whitespace
|
|
//! let tokens = lex("mi prami do");
|
|
//!
|
|
//! assert_eq![
|
|
//! tokens.iter().map(|t| t.s ).collect::<Vec<&str>>(),
|
|
//! ["mi", " ", "prami", " ", "do"]
|
|
//! ];
|
|
//!
|
|
//! // Lex without whitespace
|
|
//! let stripped_tokens = lex_stripped("ti barda gerku");
|
|
//!
|
|
//! assert_eq![
|
|
//! stripped_tokens.iter().map(|t| t.s ).collect::<Vec<&str>>(),
|
|
//! ["ti", "barda", "gerku"]
|
|
//! ];
|
|
//! ```
|
|
|
|
mod lex;
|
|
pub mod lojbanic;
|
|
mod strange;
|
|
pub use lex::*;
|
|
use pretty_assertions::{assert_eq, assert_ne};
|
|
|
|
#[derive(Debug, Eq, PartialEq)]
|
|
pub struct Token<'src_buf> {
|
|
pub s: &'src_buf str,
|
|
pub kind: TokenKind,
|
|
}
|
|
|
|
macro_rules! gen_token_fns {
|
|
($($nym:ident, $v:ident),*) => {$(
|
|
pub fn $nym(s: &'src_buf str) -> Self {
|
|
Self {
|
|
s,
|
|
kind: TokenKind::$v,
|
|
}
|
|
})*
|
|
};
|
|
}
|
|
|
|
impl<'src_buf> Token<'src_buf> {
|
|
gen_token_fns![
|
|
brivla, Brivla, cmavo, Cmavo, cmevla, Cmevla, number, Number, unknown, Unknown, whitespace,
|
|
Whitespace
|
|
];
|
|
}
|
|
|
|
#[derive(Debug, Eq, PartialEq)]
|
|
pub enum TokenKind {
|
|
Brivla,
|
|
Cmavo,
|
|
Cmevla,
|
|
Number,
|
|
Unknown,
|
|
Whitespace,
|
|
}
|
|
|
|
#[test]
|
|
fn lexes() {
|
|
assert_eq![
|
|
lex("mi prami do"),
|
|
[
|
|
Token::cmavo("mi"),
|
|
Token::whitespace(" "),
|
|
Token::brivla("prami"),
|
|
Token::whitespace(" "),
|
|
Token::cmavo("do")
|
|
]
|
|
];
|
|
assert_eq![
|
|
lex("miprAmido"),
|
|
[
|
|
Token::cmavo("mi"),
|
|
Token::brivla("prAmi"),
|
|
Token::cmavo("do")
|
|
]
|
|
];
|
|
assert_eq![lex("garbage"), [Token::brivla("garbage")]];
|
|
assert_eq![lex("loprami"), [Token::cmavo("lo"), Token::brivla("prami")]];
|
|
assert_eq![
|
|
lex("mablabigerku"),
|
|
[Token::cmavo("ma"), Token::brivla("blabigerku")]
|
|
];
|
|
assert_eq![
|
|
lex("tosmabru"),
|
|
[Token::cmavo("to"), Token::brivla("smabru")]
|
|
];
|
|
assert_eq![
|
|
lex("to mi puza zbasu lo me'oi server mu'i lo nu nu'onai zbasu lo vi se irci"),
|
|
[
|
|
Token::cmavo("to"),
|
|
Token::whitespace(" "),
|
|
Token::cmavo("mi"),
|
|
Token::whitespace(" "),
|
|
Token::cmavo("pu"),
|
|
Token::cmavo("za"),
|
|
Token::whitespace(" "),
|
|
Token::brivla("zbasu"),
|
|
Token::whitespace(" "),
|
|
Token::cmavo("lo"),
|
|
Token::whitespace(" "),
|
|
Token::cmavo("me'oi"),
|
|
Token::whitespace(" "),
|
|
Token::cmevla("server"),
|
|
Token::whitespace(" "),
|
|
Token::cmavo("mu'i"),
|
|
Token::whitespace(" "),
|
|
Token::cmavo("lo"),
|
|
Token::whitespace(" "),
|
|
Token::cmavo("nu"),
|
|
Token::whitespace(" "),
|
|
Token::cmavo("nu'o"),
|
|
Token::cmavo("nai"),
|
|
Token::whitespace(" "),
|
|
Token::brivla("zbasu"),
|
|
Token::whitespace(" "),
|
|
Token::cmavo("lo"),
|
|
Token::whitespace(" "),
|
|
Token::cmavo("vi"),
|
|
Token::whitespace(" "),
|
|
Token::cmavo("se"),
|
|
Token::whitespace(" "),
|
|
Token::brivla("irci")
|
|
]
|
|
];
|
|
}
|