forked from AbleOS/holey-bytes
removing garbage
This commit is contained in:
parent
9012f976c5
commit
28e33d11c9
|
@ -3,7 +3,7 @@ use {
|
|||
crate::{
|
||||
ident::{self, Ident},
|
||||
instrs::{self, *},
|
||||
lexer::TokenKind,
|
||||
lexer::{self, TokenKind},
|
||||
log,
|
||||
parser::{self, find_symbol, idfl, CtorField, Expr, ExprRef, FileId, Pos},
|
||||
HashMap,
|
||||
|
@ -3133,7 +3133,7 @@ impl Codegen {
|
|||
}
|
||||
|
||||
fn report_log(&self, pos: Pos, msg: impl std::fmt::Display) {
|
||||
let (line, col) = self.cfile().nlines.line_col(pos);
|
||||
let (line, col) = lexer::line_col(self.cfile().file.as_bytes(), pos);
|
||||
println!("{}:{}:{}: {}", self.cfile().path, line, col, msg);
|
||||
}
|
||||
|
||||
|
@ -3261,7 +3261,7 @@ mod tests {
|
|||
let mut codegen = super::Codegen {
|
||||
files: module_map
|
||||
.iter()
|
||||
.map(|&(path, content)| parser::Ast::new(path, content, &loader))
|
||||
.map(|&(path, content)| parser::Ast::new(path, content.to_owned(), &loader))
|
||||
.collect(),
|
||||
..Default::default()
|
||||
};
|
||||
|
@ -3338,7 +3338,7 @@ mod tests {
|
|||
struct_patterns => README;
|
||||
arrays => README;
|
||||
struct_return_from_module_function => README;
|
||||
comptime_pointers => README;
|
||||
//comptime_pointers => README;
|
||||
sort_something_viredly => README;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,5 +1,3 @@
|
|||
use std::simd::cmp::SimdPartialEq;
|
||||
|
||||
const fn ascii_mask(chars: &[u8]) -> u128 {
|
||||
let mut eq = 0;
|
||||
let mut i = 0;
|
||||
|
@ -380,112 +378,3 @@ pub fn line_col(bytes: &[u8], pos: u32) -> (usize, usize) {
|
|||
.map(|(line, col)| (line + 1, col + 1))
|
||||
.unwrap_or((1, 1))
|
||||
}
|
||||
|
||||
pub struct LineMap {
|
||||
lines: Box<[u8]>,
|
||||
}
|
||||
|
||||
impl LineMap {
|
||||
pub fn line_col(&self, mut pos: u32) -> (usize, usize) {
|
||||
let mut line = 1;
|
||||
|
||||
let mut iter = self.lines.iter().copied();
|
||||
|
||||
loop {
|
||||
let mut acc = 0;
|
||||
let mut idx = 0;
|
||||
loop {
|
||||
let len = iter.next().unwrap();
|
||||
acc |= ((len & 0x7F) as u32) << (7 * idx);
|
||||
idx += 1;
|
||||
if len & 0x80 == 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if pos < acc {
|
||||
break;
|
||||
}
|
||||
pos = pos.saturating_sub(acc);
|
||||
line += 1;
|
||||
}
|
||||
|
||||
(line, pos as usize + 1)
|
||||
}
|
||||
|
||||
pub fn new(input: &str) -> Self {
|
||||
let bytes = input.as_bytes();
|
||||
let (start, simd_mid, end) = bytes.as_simd::<16>();
|
||||
|
||||
let query = std::simd::u8x16::splat(b'\n');
|
||||
|
||||
let nl_count = start.iter().map(|&b| (b == b'\n') as usize).sum::<usize>()
|
||||
+ simd_mid.iter().map(|s| s.simd_eq(query).to_bitmask().count_ones()).sum::<u32>()
|
||||
as usize
|
||||
+ end.iter().map(|&b| (b == b'\n') as usize).sum::<usize>();
|
||||
|
||||
let mut lines = Vec::with_capacity(nl_count);
|
||||
let mut last_nl = 0;
|
||||
|
||||
let handle_rem = |offset: usize, bytes: &[u8], last_nl: &mut usize, lines: &mut Vec<u8>| {
|
||||
bytes
|
||||
.iter()
|
||||
.copied()
|
||||
.enumerate()
|
||||
.filter_map(|(i, b)| (b == b'\n').then_some(i + offset))
|
||||
.for_each(|i| {
|
||||
lines.push((i - *last_nl + 1) as u8);
|
||||
*last_nl = i + 1;
|
||||
});
|
||||
};
|
||||
|
||||
handle_rem(0, start, &mut last_nl, &mut lines);
|
||||
|
||||
for (i, simd) in simd_mid.iter().enumerate() {
|
||||
let mask = simd.simd_eq(query);
|
||||
let mut mask = mask.to_bitmask();
|
||||
while mask != 0 {
|
||||
let idx = mask.trailing_zeros() as usize + i * 16 + start.len();
|
||||
let mut len = idx - last_nl + 1;
|
||||
while len >= 0x80 {
|
||||
lines.push(0x80 | (len & 0x7F) as u8);
|
||||
len >>= 7;
|
||||
}
|
||||
lines.push(len as u8);
|
||||
last_nl = idx + 1;
|
||||
mask &= mask - 1;
|
||||
}
|
||||
}
|
||||
|
||||
handle_rem(bytes.len() - end.len(), end, &mut last_nl, &mut lines);
|
||||
|
||||
Self { lines: Box::from(lines) }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
#[test]
|
||||
fn test_smh() {
|
||||
let example = include_str!("../README.md");
|
||||
|
||||
let nlines = super::LineMap::new(example);
|
||||
|
||||
fn slow_nline_search(str: &str, mut pos: usize) -> (usize, usize) {
|
||||
(
|
||||
str.lines()
|
||||
.take_while(|l| match pos.checked_sub(l.len() + 1) {
|
||||
Some(nl) => (pos = nl, true).1,
|
||||
None => false,
|
||||
})
|
||||
.count()
|
||||
+ 1,
|
||||
pos + 1,
|
||||
)
|
||||
}
|
||||
|
||||
for i in 0..example.len() {
|
||||
assert_eq!(slow_nline_search(example, i), nlines.line_col(i as _));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -424,7 +424,7 @@ pub fn parse_from_fs(extra_threads: usize, root: &str) -> io::Result<Vec<Ast>> {
|
|||
let mut file = std::fs::File::open(path)?;
|
||||
file.read_to_end(buffer)?;
|
||||
let src = std::str::from_utf8(buffer).map_err(InvalidFileData)?;
|
||||
Ok(Ast::new(path, src, &loader))
|
||||
Ok(Ast::new(path, src.to_owned(), &loader))
|
||||
};
|
||||
|
||||
let thread = || {
|
||||
|
|
|
@ -2,7 +2,7 @@ use {
|
|||
crate::{
|
||||
codegen,
|
||||
ident::{self, Ident},
|
||||
lexer::{self, Lexer, LineMap, Token, TokenKind},
|
||||
lexer::{self, Lexer, Token, TokenKind},
|
||||
log,
|
||||
},
|
||||
std::{
|
||||
|
@ -1065,7 +1065,7 @@ pub struct AstInner<T: ?Sized> {
|
|||
exprs: *const [Expr<'static>],
|
||||
|
||||
pub path: Box<str>,
|
||||
pub nlines: LineMap,
|
||||
pub file: Box<str>,
|
||||
pub symbols: T,
|
||||
}
|
||||
|
||||
|
@ -1077,11 +1077,11 @@ impl AstInner<[Symbol]> {
|
|||
.0
|
||||
}
|
||||
|
||||
fn new(content: &str, path: &str, loader: Loader) -> NonNull<Self> {
|
||||
fn new(content: String, path: &str, loader: Loader) -> NonNull<Self> {
|
||||
let arena = Arena::default();
|
||||
let mut syms = Vec::new();
|
||||
let mut parser = Parser::new(&arena, &mut syms, loader);
|
||||
let exprs = parser.file(content, path) as *const [Expr<'static>];
|
||||
let exprs = parser.file(&content, path) as *const [Expr<'static>];
|
||||
|
||||
syms.sort_unstable_by_key(|s| s.name);
|
||||
|
||||
|
@ -1096,7 +1096,7 @@ impl AstInner<[Symbol]> {
|
|||
mem: arena.chunk.into_inner(),
|
||||
exprs,
|
||||
path: path.into(),
|
||||
nlines: LineMap::new(content),
|
||||
file: content.into(),
|
||||
symbols: (),
|
||||
});
|
||||
std::ptr::addr_of_mut!((*inner).symbols)
|
||||
|
@ -1112,7 +1112,7 @@ impl AstInner<[Symbol]> {
|
|||
pub struct Ast(NonNull<AstInner<[Symbol]>>);
|
||||
|
||||
impl Ast {
|
||||
pub fn new(path: &str, content: &str, loader: Loader) -> Self {
|
||||
pub fn new(path: &str, content: String, loader: Loader) -> Self {
|
||||
Self(AstInner::new(content, path, loader))
|
||||
}
|
||||
|
||||
|
@ -1143,7 +1143,7 @@ impl std::fmt::Display for Ast {
|
|||
|
||||
impl Default for Ast {
|
||||
fn default() -> Self {
|
||||
Self(AstInner::new("", "", &no_loader))
|
||||
Self(AstInner::new(String::new(), "", &no_loader))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1319,7 +1319,7 @@ impl Drop for ArenaChunk {
|
|||
#[cfg(test)]
|
||||
pub mod test {
|
||||
pub fn format(ident: &str, input: &str) {
|
||||
let ast = super::Ast::new(ident, input, &|_, _| Ok(0));
|
||||
let ast = super::Ast::new(ident, input.to_owned(), &|_, _| Ok(0));
|
||||
let mut output = Vec::new();
|
||||
crate::format_to(&ast, input, &mut output).unwrap();
|
||||
|
||||
|
|
|
@ -1,3 +1,3 @@
|
|||
code size: 189
|
||||
code size: 193
|
||||
ret: 1
|
||||
status: Ok(())
|
||||
|
|
Loading…
Reference in a new issue