holey-bytes/hblang/src/lexer.rs

510 lines
13 KiB
Rust
Raw Normal View History

2024-06-25 12:55:25 -05:00
use std::simd::cmp::SimdPartialEq;
2024-05-19 11:20:42 -05:00
2024-06-25 11:39:59 -05:00
const fn ascii_mask(chars: &[u8]) -> u128 {
let mut eq = 0;
let mut i = 0;
while i < chars.len() {
let b = chars[i];
eq |= 1 << b;
i += 1;
}
eq
}
2024-05-11 09:04:13 -05:00
#[derive(Debug, PartialEq, Eq, Clone, Copy)]
2024-05-09 16:41:59 -05:00
pub struct Token {
2024-07-08 00:22:53 -05:00
pub kind: TokenKind,
2024-05-09 16:41:59 -05:00
pub start: u32,
2024-07-08 00:22:53 -05:00
pub end: u32,
2024-05-09 16:41:59 -05:00
}
impl Token {
pub fn range(&self) -> std::ops::Range<usize> {
self.start as usize..self.end as usize
}
}
2024-05-11 15:22:08 -05:00
macro_rules! gen_token_kind {
($(
#[$atts:meta])*
$vis:vis enum $name:ident {
#[patterns] $(
$pattern:ident,
)*
#[keywords] $(
$keyword:ident = $keyword_lit:literal,
)*
#[punkt] $(
$punkt:ident = $punkt_lit:literal,
)*
#[ops] $(
2024-05-15 03:37:39 -05:00
#[$prec:ident] $(
$op:ident = $op_lit:literal $(=> $assign:ident)?,
2024-05-11 15:22:08 -05:00
)*
)*
}
) => {
impl std::fmt::Display for $name {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
2024-06-25 11:39:59 -05:00
let sf = *self as u8;
f.write_str(match *self {
2024-05-11 15:22:08 -05:00
$( Self::$pattern => concat!('<', stringify!($pattern), '>'), )*
$( Self::$keyword => stringify!($keyword_lit), )*
$( Self::$punkt => stringify!($punkt_lit), )*
2024-05-15 03:37:39 -05:00
$($( Self::$op => $op_lit,
$(Self::$assign => concat!($op_lit, "="),)?)*)*
2024-06-25 11:39:59 -05:00
_ => unsafe { std::str::from_utf8_unchecked(std::slice::from_ref(&sf)) },
})
2024-05-11 15:22:08 -05:00
}
}
2024-05-09 16:41:59 -05:00
2024-05-11 15:22:08 -05:00
impl $name {
#[inline(always)]
pub fn precedence(&self) -> Option<u8> {
Some(match self {
2024-05-15 03:37:39 -05:00
$($(Self::$op => ${ignore($prec)} ${index(1)},
$(Self::$assign => 0,)?)*)*
2024-05-11 15:22:08 -05:00
_ => return None,
2024-05-15 03:37:39 -05:00
} + 1)
2024-05-11 15:22:08 -05:00
}
#[inline(always)]
fn from_ident(ident: &[u8]) -> Self {
match ident {
$($keyword_lit => Self::$keyword,)*
_ => Self::Ident,
}
}
}
2024-06-25 11:39:59 -05:00
};
}
#[derive(Debug, PartialEq, Eq, Clone, Copy)]
#[repr(u8)]
pub enum TokenKind {
2024-07-08 00:22:53 -05:00
Not = b'!',
DQuote = b'"',
Pound = b'#',
2024-06-25 11:39:59 -05:00
CtIdent = b'$',
2024-07-08 00:22:53 -05:00
Mod = b'%',
Band = b'&',
Quote = b'\'',
LParen = b'(',
RParen = b')',
Mul = b'*',
Add = b'+',
Comma = b',',
Sub = b'-',
Dot = b'.',
Div = b'/',
2024-06-25 12:12:35 -05:00
// Unused = 2-6
Shl = b'<' - 5,
2024-06-25 12:12:35 -05:00
// Unused = 8
Shr = b'>' - 5,
2024-07-08 00:22:53 -05:00
Colon = b':',
Semi = b';',
Lt = b'<',
Assign = b'=',
Gt = b'>',
Que = b'?',
2024-06-25 11:39:59 -05:00
Directive = b'@',
2024-06-25 12:46:48 -05:00
Comment,
2024-06-25 11:39:59 -05:00
Ident,
Number,
Eof,
2024-05-11 15:22:08 -05:00
Ct,
2024-06-25 11:39:59 -05:00
Return,
If,
Else,
Loop,
Break,
Continue,
Fn,
Struct,
True,
Ctor,
Tupl,
Or,
And,
// Unused = R-Z
2024-07-08 00:22:53 -05:00
LBrack = b'[',
BSlash = b'\\',
RBrack = b']',
Xor = b'^',
Tick = b'`',
2024-06-25 11:39:59 -05:00
// Unused = a-z
2024-07-08 00:22:53 -05:00
LBrace = b'{',
Bor = b'|',
RBrace = b'}',
Tilde = b'~',
Decl = b':' + 128,
Eq = b'=' + 128,
Ne = b'!' + 128,
Le = b'<' + 128,
Ge = b'>' + 128,
BorAss = b'|' + 128,
AddAss = b'+' + 128,
SubAss = b'-' + 128,
MulAss = b'*' + 128,
DivAss = b'/' + 128,
ModAss = b'%' + 128,
XorAss = b'^' + 128,
2024-06-25 11:39:59 -05:00
BandAss = b'&' + 128,
ShrAss = b'>' - 5 + 128,
ShlAss = b'<' - 5 + 128,
2024-06-25 11:39:59 -05:00
}
impl TokenKind {
2024-07-08 00:22:53 -05:00
pub fn ass_op(self) -> Option<Self> {
2024-06-25 11:39:59 -05:00
let id = (self as u8).saturating_sub(128);
if ascii_mask(b"|+-*/%^&79") & (1u128 << id) == 0 {
2024-06-25 11:39:59 -05:00
return None;
2024-05-11 15:22:08 -05:00
}
2024-06-25 11:39:59 -05:00
Some(unsafe { std::mem::transmute::<u8, Self>(id) })
}
2024-05-10 15:54:12 -05:00
}
2024-05-11 15:22:08 -05:00
gen_token_kind! {
pub enum TokenKind {
#[patterns]
2024-06-01 13:30:07 -05:00
CtIdent,
2024-05-11 15:22:08 -05:00
Ident,
Number,
Eof,
2024-06-25 11:39:59 -05:00
Directive,
2024-05-11 15:22:08 -05:00
#[keywords]
Return = b"return",
If = b"if",
Else = b"else",
Loop = b"loop",
Break = b"break",
Continue = b"continue",
Fn = b"fn",
2024-05-12 05:16:40 -05:00
Struct = b"struct",
2024-05-12 13:10:50 -05:00
True = b"true",
2024-05-11 15:22:08 -05:00
#[punkt]
2024-05-12 05:16:40 -05:00
Ctor = ".{",
2024-05-15 03:37:39 -05:00
Tupl = ".(",
2024-05-11 15:22:08 -05:00
#[ops]
2024-05-15 03:37:39 -05:00
#[prec]
2024-05-12 05:16:40 -05:00
Decl = ":=",
2024-05-11 15:22:08 -05:00
Assign = "=",
2024-05-15 03:37:39 -05:00
#[prec]
Or = "||",
#[prec]
And = "&&",
#[prec]
Bor = "|" => BorAss,
#[prec]
Xor = "^" => XorAss,
#[prec]
Band = "&" => BandAss,
#[prec]
Eq = "==",
Ne = "!=",
#[prec]
2024-05-11 15:22:08 -05:00
Le = "<=",
2024-05-13 06:36:29 -05:00
Ge = ">=",
Lt = "<",
Gt = ">",
2024-05-15 03:37:39 -05:00
#[prec]
Shl = "<<" => ShlAss,
Shr = ">>" => ShrAss,
#[prec]
Add = "+" => AddAss,
Sub = "-" => SubAss,
#[prec]
Mul = "*" => MulAss,
Div = "/" => DivAss,
Mod = "%" => ModAss,
2024-05-10 15:54:12 -05:00
}
}
2024-05-09 16:41:59 -05:00
pub struct Lexer<'a> {
2024-07-08 00:22:53 -05:00
pos: u32,
2024-05-09 16:41:59 -05:00
bytes: &'a [u8],
}
impl<'a> Lexer<'a> {
pub fn new(input: &'a str) -> Self {
2024-06-24 10:26:00 -05:00
Self::restore(input, 0)
}
pub fn restore(input: &'a str, pos: u32) -> Self {
2024-07-08 00:22:53 -05:00
Self { pos, bytes: input.as_bytes() }
2024-05-09 16:41:59 -05:00
}
2024-05-12 04:52:58 -05:00
pub fn slice(&self, tok: std::ops::Range<usize>) -> &'a str {
unsafe { std::str::from_utf8_unchecked(&self.bytes[tok]) }
2024-05-09 16:41:59 -05:00
}
fn peek(&self) -> Option<u8> {
2024-06-25 12:12:35 -05:00
if std::intrinsics::unlikely(self.pos >= self.bytes.len() as u32) {
None
} else {
Some(unsafe { *self.bytes.get_unchecked(self.pos as usize) })
}
2024-05-09 16:41:59 -05:00
}
fn advance(&mut self) -> Option<u8> {
let c = self.peek()?;
self.pos += 1;
Some(c)
}
2024-07-19 14:04:22 -05:00
pub fn last(&mut self) -> Token {
let mut token = self.next();
loop {
let next = self.next();
if next.kind == TokenKind::Eof {
break;
}
token = next;
}
token
}
2024-05-09 16:41:59 -05:00
pub fn next(&mut self) -> Token {
2024-05-15 03:37:39 -05:00
use TokenKind as T;
loop {
let mut start = self.pos;
let Some(c) = self.advance() else {
2024-07-08 00:22:53 -05:00
return Token { kind: T::Eof, start, end: self.pos };
2024-05-15 03:37:39 -05:00
};
2024-06-01 13:30:07 -05:00
let advance_ident = |s: &mut Self| {
2024-06-25 11:39:59 -05:00
while let Some(b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'_' | 127..) = s.peek() {
2024-06-01 13:30:07 -05:00
s.advance();
}
};
2024-06-25 12:12:35 -05:00
let identity = |s: u8| unsafe { std::mem::transmute::<u8, T>(s) };
let kind = match c {
2024-06-25 11:39:59 -05:00
..=b' ' => continue,
2024-09-01 12:42:04 -05:00
b'0' if self.advance_if(b'x') => {
while let Some(b'0'..=b'9' | b'A'..=b'F' | b'a'..=b'f') = self.peek() {
2024-09-01 12:42:04 -05:00
self.advance();
}
T::Number
}
b'0' if self.advance_if(b'b') => {
while let Some(b'0' | b'1') = self.peek() {
2024-09-01 12:42:04 -05:00
self.advance();
}
T::Number
}
2024-09-01 12:45:38 -05:00
b'0' if self.advance_if(b'o') => {
while let Some(b'0'..=b'7') = self.peek() {
2024-09-01 12:45:38 -05:00
self.advance();
}
T::Number
}
2024-05-15 03:37:39 -05:00
b'0'..=b'9' => {
while let Some(b'0'..=b'9') = self.peek() {
self.advance();
}
T::Number
}
2024-06-25 11:39:59 -05:00
b'a'..=b'z' | b'A'..=b'Z' | b'_' | 127.. => {
2024-06-01 13:30:07 -05:00
advance_ident(self);
let ident = &self.bytes[start as usize..self.pos as usize];
T::from_ident(ident)
2024-05-15 03:37:39 -05:00
}
2024-06-25 12:46:48 -05:00
b'"' | b'\'' => loop {
match self.advance() {
Some(b'\\') => _ = self.advance(),
Some(nc) if nc == c => break identity(c),
Some(_) => {}
None => break T::Eof,
}
},
b'/' if self.advance_if(b'/') => {
while let Some(l) = self.advance()
&& l != b'\n'
{}
T::Comment
}
b'/' if self.advance_if(b'*') => {
let mut depth = 1;
while let Some(l) = self.advance() {
match l {
b'/' if self.advance_if(b'*') => depth += 1,
b'*' if self.advance_if(b'/') => match depth {
1 => break,
_ => depth -= 1,
},
2024-05-17 12:53:59 -05:00
_ => {}
}
}
2024-06-25 12:46:48 -05:00
T::Comment
2024-05-17 12:53:59 -05:00
}
2024-05-15 03:37:39 -05:00
b'.' if self.advance_if(b'{') => T::Ctor,
b'.' if self.advance_if(b'(') => T::Tupl,
2024-06-25 12:22:49 -05:00
b'&' if self.advance_if(b'&') => T::And,
b'|' if self.advance_if(b'|') => T::Or,
b'$' if self.advance_if(b':') => T::Ct,
2024-06-25 12:22:49 -05:00
b'@' | b'$' => {
start += 1;
advance_ident(self);
identity(c)
}
2024-06-25 12:12:35 -05:00
b'<' | b'>' if self.advance_if(c) => {
identity(c - 5 + 128 * self.advance_if(b'=') as u8)
}
b':' | b'=' | b'!' | b'<' | b'>' | b'|' | b'+' | b'-' | b'*' | b'/' | b'%'
| b'^' | b'&'
if self.advance_if(b'=') =>
{
identity(c + 128)
}
_ => identity(c),
2024-05-15 03:37:39 -05:00
};
2024-07-08 00:22:53 -05:00
return Token { kind, start, end: self.pos };
2024-05-15 03:37:39 -05:00
}
2024-05-09 16:41:59 -05:00
}
fn advance_if(&mut self, arg: u8) -> bool {
if self.peek() == Some(arg) {
self.advance();
true
} else {
false
}
}
2024-05-12 16:19:45 -05:00
pub fn line_col(&self, pos: u32) -> (usize, usize) {
line_col(self.bytes, pos)
2024-05-09 16:41:59 -05:00
}
}
2024-05-19 11:20:42 -05:00
pub fn line_col(bytes: &[u8], pos: u32) -> (usize, usize) {
bytes[..pos as usize]
2024-05-12 16:19:45 -05:00
.split(|&b| b == b'\n')
2024-05-19 11:20:42 -05:00
.map(<[u8]>::len)
2024-05-12 16:19:45 -05:00
.enumerate()
2024-05-19 11:20:42 -05:00
.last()
.map(|(line, col)| (line + 1, col + 1))
2024-05-12 16:19:45 -05:00
.unwrap_or((1, 1))
}
2024-05-19 11:20:42 -05:00
pub struct LineMap {
lines: Box<[u8]>,
}
2024-05-09 16:41:59 -05:00
2024-05-19 11:20:42 -05:00
impl LineMap {
pub fn line_col(&self, mut pos: u32) -> (usize, usize) {
let mut line = 1;
2024-05-09 16:41:59 -05:00
2024-05-19 11:20:42 -05:00
let mut iter = self.lines.iter().copied();
2024-06-24 10:26:00 -05:00
loop {
2024-05-19 11:20:42 -05:00
let mut acc = 0;
2024-06-24 10:26:00 -05:00
let mut idx = 0;
loop {
let len = iter.next().unwrap();
acc |= ((len & 0x7F) as u32) << (7 * idx);
idx += 1;
if len & 0x80 == 0 {
break;
}
2024-05-19 11:20:42 -05:00
}
if pos < acc {
break;
}
pos = pos.saturating_sub(acc);
line += 1;
}
(line, pos as usize + 1)
}
pub fn new(input: &str) -> Self {
let bytes = input.as_bytes();
let (start, simd_mid, end) = bytes.as_simd::<16>();
let query = std::simd::u8x16::splat(b'\n');
let nl_count = start.iter().map(|&b| (b == b'\n') as usize).sum::<usize>()
2024-07-08 00:22:53 -05:00
+ simd_mid.iter().map(|s| s.simd_eq(query).to_bitmask().count_ones()).sum::<u32>()
as usize
2024-05-19 11:20:42 -05:00
+ end.iter().map(|&b| (b == b'\n') as usize).sum::<usize>();
let mut lines = Vec::with_capacity(nl_count);
let mut last_nl = 0;
let handle_rem = |offset: usize, bytes: &[u8], last_nl: &mut usize, lines: &mut Vec<u8>| {
bytes
.iter()
.copied()
.enumerate()
.filter_map(|(i, b)| (b == b'\n').then_some(i + offset))
.for_each(|i| {
lines.push((i - *last_nl + 1) as u8);
*last_nl = i + 1;
});
};
handle_rem(0, start, &mut last_nl, &mut lines);
for (i, simd) in simd_mid.iter().enumerate() {
let mask = simd.simd_eq(query);
let mut mask = mask.to_bitmask();
while mask != 0 {
let idx = mask.trailing_zeros() as usize + i * 16 + start.len();
let mut len = idx - last_nl + 1;
while len >= 0x80 {
2024-06-24 10:26:00 -05:00
lines.push(0x80 | (len & 0x7F) as u8);
2024-05-19 11:20:42 -05:00
len >>= 7;
2024-05-09 16:41:59 -05:00
}
2024-05-19 11:20:42 -05:00
lines.push(len as u8);
last_nl = idx + 1;
mask &= mask - 1;
}
}
2024-05-09 16:41:59 -05:00
2024-05-19 11:20:42 -05:00
handle_rem(bytes.len() - end.len(), end, &mut last_nl, &mut lines);
2024-07-08 00:22:53 -05:00
Self { lines: Box::from(lines) }
2024-05-09 16:41:59 -05:00
}
}
2024-06-24 10:26:00 -05:00
#[cfg(test)]
mod test {
#[test]
fn test_smh() {
let example = include_str!("../README.md");
let nlines = super::LineMap::new(example);
fn slow_nline_search(str: &str, mut pos: usize) -> (usize, usize) {
(
str.lines()
.take_while(|l| match pos.checked_sub(l.len() + 1) {
Some(nl) => (pos = nl, true).1,
None => false,
})
.count()
+ 1,
pos + 1,
)
}
for i in 0..example.len() {
assert_eq!(slow_nline_search(example, i), nlines.line_col(i as _));
}
}
}